2024-09-21T03:53:59.524 INFO:root:teuthology version: 1.2.2.dev2+g3752d38 2024-09-21T03:53:59.542 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2024-09-21T03:53:59.629 INFO:teuthology.run:Config: archive_path: /home/teuthworker/archive/gabrioux-2024-09-21_03:44:43-orch:cephadm-wip-guits-main-2024-09-20-2310-distro-default-smithi/7913679 branch: wip-guits-main-2024-09-20-2310 description: orch:cephadm/mgr-nfs-upgrade/{0-centos_9.stream 1-bootstrap/17.2.0 1-start 2-nfs 3-upgrade-with-workload 4-final} email: gabrioux@ibm.com first_in_suite: false job_id: '7913679' kernel: kdb: 1 sha1: distro last_in_suite: false machine_type: smithi name: gabrioux-2024-09-21_03:44:43-orch:cephadm-wip-guits-main-2024-09-20-2310-distro-default-smithi no_nested_subset: false openstack: - volumes: count: 4 size: 10 os_type: centos os_version: 9.stream overrides: admin_socket: branch: wip-guits-main-2024-09-20-2310 ceph: conf: mgr: debug mgr: 20 debug ms: 1 mon: debug mon: 20 debug ms: 1 debug paxos: 20 osd: debug ms: 1 debug osd: 20 osd shutdown pgref assert: true flavor: default log-ignorelist: - \(MDS_ALL_DOWN\) - \(MDS_UP_LESS_THAN_MAX\) - CEPHADM_REFRESH_FAILED log-only-match: - CEPHADM_ sha1: f36283e7c91cac1c01cb9a7c4ff21bd697dbd4f2 ceph-deploy: conf: client: log file: /var/log/ceph/ceph-$name.$pid.log mon: {} install: ceph: flavor: default sha1: f36283e7c91cac1c01cb9a7c4ff21bd697dbd4f2 selinux: allowlist: - scontext=system_u:system_r:logrotate_t:s0 workunit: branch: wip-guits-main-2024-09-20-2310 sha1: f36283e7c91cac1c01cb9a7c4ff21bd697dbd4f2 owner: scheduled_gabrioux@teuthology priority: 70 repo: https://git.ceph.com/ceph-ci.git roles: - - host.a - osd.0 - osd.1 - osd.2 - osd.3 - client.0 - - host.b - osd.4 - osd.5 - osd.6 - osd.7 seed: 3363 sha1: f36283e7c91cac1c01cb9a7c4ff21bd697dbd4f2 sleep_before_teardown: 0 subset: 1/8 suite: orch:cephadm suite_branch: wip-guits-main-2024-09-20-2310 suite_path: /home/teuthworker/src/git.ceph.com_ceph-c_f36283e7c91cac1c01cb9a7c4ff21bd697dbd4f2/qa suite_relpath: qa suite_repo: https://git.ceph.com/ceph-ci.git suite_sha1: f36283e7c91cac1c01cb9a7c4ff21bd697dbd4f2 targets: smithi089.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBD3uyXfifXK/TbqZii59GHshc4PHZglCYTQbT6aSNKXO02UtcXCZ7c8hLNKzxd3F+o28cIbRqFygCMbFQuXqdx4= smithi195.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBPQnd2IQicJjMzAl8XFjFQoU2lM+W9O8g3UuOwa+OrcDOFsa5VVbK5zXKD8ak2q3Qt101HzaJydMAFq3xGUpUvY= tasks: - cephadm: cephadm_branch: v17.2.0 cephadm_git_url: https://github.com/ceph/ceph image: quay.io/ceph/ceph:v17.2.0 roleless: true - cephadm.shell: host.a: - ceph orch status - ceph orch ps - ceph orch ls - ceph orch host ls - ceph orch device ls - vip.exec: all-hosts: - systemctl stop nfs-server - cephadm.shell: host.a: - ceph fs volume create foofs - cephadm.wait_for_service: service: mds.foofs - cephadm.shell: host.a: - ceph nfs cluster create foo --placement=2 || ceph nfs cluster create cephfs foo --placement=2 - ceph nfs export create cephfs --fsname foofs --clusterid foo --binding /fake || ceph nfs export create cephfs --fsname foofs --cluster-id foo --pseudo-path /fake - while ! ceph orch ls | grep nfs | grep 2/2 ; do sleep 1 ; done - vip.exec: host.a: - mkdir /mnt/foo - while ! mount -t nfs $(hostname):/fake /mnt/foo -o sync ; do sleep 5 ; done - echo test > /mnt/foo/testfile - sync - parallel: - upgrade-tasks - workload-tasks - vip.exec: host.a: - umount /mnt/foo - cephadm.shell: host.a: - ceph nfs cluster ls | grep foo - ceph nfs export ls foo --detailed - rados -p .nfs --all ls - - cephadm.shell: host.a: - 'set -ex [[ `ceph config get mgr mgr/cephadm/migration_current` -gt 2 ]] ' teuthology: fragments_dropped: [] meta: {} postmerge: [] teuthology_branch: main teuthology_sha1: 3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a timestamp: 2024-09-21_03:44:43 tube: smithi upgrade-tasks: sequential: - cephadm.shell: env: - sha1 host.a: - ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force - ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force - ceph config set global log_to_journald false --force - ceph mgr module enable nfs --force - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 - cephadm.shell: env: - sha1 host.a: - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; ceph health detail ; sleep 30 ; done - ceph orch ps - ceph versions - echo "wait for servicemap items w/ changing names to refresh" - sleep 60 - ceph orch ps - ceph orch upgrade status - ceph health detail - ceph versions - ceph versions | jq -e '.overall | length == 1' - ceph versions | jq -e '.overall | keys' | grep $sha1 - cephadm.wait_for_service: service: nfs.foo user: gabrioux verbose: true worker_log: /home/teuthworker/archive/worker_logs/dispatcher.smithi.2902 workload-tasks: sequential: - exec: host.a: - cd /mnt/foo && dbench 5 -t 600 || true - umount /mnt/foo - while ! mount -t nfs $(hostname):/fake /mnt/foo ; do sleep 5 ; done - cd /mnt/foo && dbench 5 -t 5 2024-09-21T03:53:59.629 INFO:teuthology.run:suite_path is set to /home/teuthworker/src/git.ceph.com_ceph-c_f36283e7c91cac1c01cb9a7c4ff21bd697dbd4f2/qa; will attempt to use it 2024-09-21T03:53:59.630 INFO:teuthology.run:Found tasks at /home/teuthworker/src/git.ceph.com_ceph-c_f36283e7c91cac1c01cb9a7c4ff21bd697dbd4f2/qa/tasks 2024-09-21T03:53:59.630 INFO:teuthology.run_tasks:Running task internal.check_packages... 2024-09-21T03:53:59.634 INFO:teuthology.task.internal:Checking packages... 2024-09-21T03:53:59.654 INFO:teuthology.task.internal:Checking packages for os_type 'centos', flavor 'default' and ceph hash 'f36283e7c91cac1c01cb9a7c4ff21bd697dbd4f2' 2024-09-21T03:53:59.654 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-09-21T03:53:59.654 INFO:teuthology.packaging:ref: None 2024-09-21T03:53:59.654 INFO:teuthology.packaging:tag: None 2024-09-21T03:53:59.654 INFO:teuthology.packaging:branch: wip-guits-main-2024-09-20-2310 2024-09-21T03:53:59.654 INFO:teuthology.packaging:sha1: f36283e7c91cac1c01cb9a7c4ff21bd697dbd4f2 2024-09-21T03:53:59.655 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&ref=wip-guits-main-2024-09-20-2310 2024-09-21T03:53:59.803 INFO:teuthology.task.internal:Found packages for ceph version 19.3.0-5103.gf36283e7 2024-09-21T03:53:59.805 INFO:teuthology.run_tasks:Running task internal.buildpackages_prep... 2024-09-21T03:53:59.811 INFO:teuthology.task.internal:no buildpackages task found 2024-09-21T03:53:59.811 INFO:teuthology.run_tasks:Running task internal.save_config... 2024-09-21T03:53:59.821 INFO:teuthology.task.internal:Saving configuration 2024-09-21T03:53:59.834 INFO:teuthology.run_tasks:Running task internal.check_lock... 2024-09-21T03:53:59.840 INFO:teuthology.task.internal.check_lock:Checking locks... 2024-09-21T03:53:59.866 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi089.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/gabrioux-2024-09-21_03:44:43-orch:cephadm-wip-guits-main-2024-09-20-2310-distro-default-smithi/7913679', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9.stream', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-09-21 03:49:02.566173', 'locked_by': 'scheduled_gabrioux@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBD3uyXfifXK/TbqZii59GHshc4PHZglCYTQbT6aSNKXO02UtcXCZ7c8hLNKzxd3F+o28cIbRqFygCMbFQuXqdx4='} 2024-09-21T03:53:59.890 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi195.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/gabrioux-2024-09-21_03:44:43-orch:cephadm-wip-guits-main-2024-09-20-2310-distro-default-smithi/7913679', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9.stream', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-09-21 03:49:02.568144', 'locked_by': 'scheduled_gabrioux@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBPQnd2IQicJjMzAl8XFjFQoU2lM+W9O8g3UuOwa+OrcDOFsa5VVbK5zXKD8ak2q3Qt101HzaJydMAFq3xGUpUvY='} 2024-09-21T03:53:59.890 INFO:teuthology.run_tasks:Running task internal.add_remotes... 2024-09-21T03:53:59.896 INFO:teuthology.task.internal:roles: ubuntu@smithi089.front.sepia.ceph.com - ['host.a', 'osd.0', 'osd.1', 'osd.2', 'osd.3', 'client.0'] 2024-09-21T03:53:59.896 INFO:teuthology.task.internal:roles: ubuntu@smithi195.front.sepia.ceph.com - ['host.b', 'osd.4', 'osd.5', 'osd.6', 'osd.7'] 2024-09-21T03:53:59.896 INFO:teuthology.run_tasks:Running task console_log... 2024-09-21T03:53:59.961 DEBUG:teuthology.exit:Installing handler: Handler(exiter=, func=.kill_console_loggers at 0x7f11aa92c280>, signals=[15]) 2024-09-21T03:53:59.961 INFO:teuthology.run_tasks:Running task internal.connect... 2024-09-21T03:53:59.967 INFO:teuthology.task.internal:Opening connections... 2024-09-21T03:53:59.967 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi089.front.sepia.ceph.com 2024-09-21T03:53:59.968 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi089.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-21T03:54:00.042 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi195.front.sepia.ceph.com 2024-09-21T03:54:00.042 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi195.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-21T03:54:00.115 INFO:teuthology.run_tasks:Running task internal.push_inventory... 2024-09-21T03:54:00.123 DEBUG:teuthology.orchestra.run.smithi089:> uname -m 2024-09-21T03:54:00.142 INFO:teuthology.orchestra.run.smithi089.stdout:x86_64 2024-09-21T03:54:00.142 DEBUG:teuthology.orchestra.run.smithi089:> cat /etc/os-release 2024-09-21T03:54:00.200 INFO:teuthology.orchestra.run.smithi089.stdout:NAME="CentOS Stream" 2024-09-21T03:54:00.201 INFO:teuthology.orchestra.run.smithi089.stdout:VERSION="9" 2024-09-21T03:54:00.201 INFO:teuthology.orchestra.run.smithi089.stdout:ID="centos" 2024-09-21T03:54:00.201 INFO:teuthology.orchestra.run.smithi089.stdout:ID_LIKE="rhel fedora" 2024-09-21T03:54:00.201 INFO:teuthology.orchestra.run.smithi089.stdout:VERSION_ID="9" 2024-09-21T03:54:00.201 INFO:teuthology.orchestra.run.smithi089.stdout:PLATFORM_ID="platform:el9" 2024-09-21T03:54:00.201 INFO:teuthology.orchestra.run.smithi089.stdout:PRETTY_NAME="CentOS Stream 9" 2024-09-21T03:54:00.201 INFO:teuthology.orchestra.run.smithi089.stdout:ANSI_COLOR="0;31" 2024-09-21T03:54:00.201 INFO:teuthology.orchestra.run.smithi089.stdout:LOGO="fedora-logo-icon" 2024-09-21T03:54:00.201 INFO:teuthology.orchestra.run.smithi089.stdout:CPE_NAME="cpe:/o:centos:centos:9" 2024-09-21T03:54:00.201 INFO:teuthology.orchestra.run.smithi089.stdout:HOME_URL="https://centos.org/" 2024-09-21T03:54:00.201 INFO:teuthology.orchestra.run.smithi089.stdout:BUG_REPORT_URL="https://issues.redhat.com/" 2024-09-21T03:54:00.201 INFO:teuthology.orchestra.run.smithi089.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 9" 2024-09-21T03:54:00.202 INFO:teuthology.orchestra.run.smithi089.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2024-09-21T03:54:00.202 INFO:teuthology.lock.ops:Updating smithi089.front.sepia.ceph.com on lock server 2024-09-21T03:54:00.221 DEBUG:teuthology.orchestra.run.smithi195:> uname -m 2024-09-21T03:54:00.238 INFO:teuthology.orchestra.run.smithi195.stdout:x86_64 2024-09-21T03:54:00.239 DEBUG:teuthology.orchestra.run.smithi195:> cat /etc/os-release 2024-09-21T03:54:00.295 INFO:teuthology.orchestra.run.smithi195.stdout:NAME="CentOS Stream" 2024-09-21T03:54:00.295 INFO:teuthology.orchestra.run.smithi195.stdout:VERSION="9" 2024-09-21T03:54:00.295 INFO:teuthology.orchestra.run.smithi195.stdout:ID="centos" 2024-09-21T03:54:00.295 INFO:teuthology.orchestra.run.smithi195.stdout:ID_LIKE="rhel fedora" 2024-09-21T03:54:00.296 INFO:teuthology.orchestra.run.smithi195.stdout:VERSION_ID="9" 2024-09-21T03:54:00.296 INFO:teuthology.orchestra.run.smithi195.stdout:PLATFORM_ID="platform:el9" 2024-09-21T03:54:00.296 INFO:teuthology.orchestra.run.smithi195.stdout:PRETTY_NAME="CentOS Stream 9" 2024-09-21T03:54:00.296 INFO:teuthology.orchestra.run.smithi195.stdout:ANSI_COLOR="0;31" 2024-09-21T03:54:00.296 INFO:teuthology.orchestra.run.smithi195.stdout:LOGO="fedora-logo-icon" 2024-09-21T03:54:00.296 INFO:teuthology.orchestra.run.smithi195.stdout:CPE_NAME="cpe:/o:centos:centos:9" 2024-09-21T03:54:00.296 INFO:teuthology.orchestra.run.smithi195.stdout:HOME_URL="https://centos.org/" 2024-09-21T03:54:00.296 INFO:teuthology.orchestra.run.smithi195.stdout:BUG_REPORT_URL="https://issues.redhat.com/" 2024-09-21T03:54:00.296 INFO:teuthology.orchestra.run.smithi195.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 9" 2024-09-21T03:54:00.296 INFO:teuthology.orchestra.run.smithi195.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2024-09-21T03:54:00.296 INFO:teuthology.lock.ops:Updating smithi195.front.sepia.ceph.com on lock server 2024-09-21T03:54:00.322 INFO:teuthology.run_tasks:Running task internal.serialize_remote_roles... 2024-09-21T03:54:00.330 INFO:teuthology.run_tasks:Running task internal.check_conflict... 2024-09-21T03:54:00.337 INFO:teuthology.task.internal:Checking for old test directory... 2024-09-21T03:54:00.338 DEBUG:teuthology.orchestra.run.smithi089:> test '!' -e /home/ubuntu/cephtest 2024-09-21T03:54:00.340 DEBUG:teuthology.orchestra.run.smithi195:> test '!' -e /home/ubuntu/cephtest 2024-09-21T03:54:00.355 INFO:teuthology.run_tasks:Running task internal.check_ceph_data... 2024-09-21T03:54:00.361 INFO:teuthology.task.internal:Checking for non-empty /var/lib/ceph... 2024-09-21T03:54:00.362 DEBUG:teuthology.orchestra.run.smithi089:> test -z $(ls -A /var/lib/ceph) 2024-09-21T03:54:00.398 DEBUG:teuthology.orchestra.run.smithi195:> test -z $(ls -A /var/lib/ceph) 2024-09-21T03:54:00.585 INFO:teuthology.run_tasks:Running task internal.vm_setup... 2024-09-21T03:54:00.642 INFO:teuthology.run_tasks:Running task kernel... 2024-09-21T03:54:00.707 INFO:teuthology.task.kernel:normalize config orig: {'kdb': 1, 'sha1': 'distro'} 2024-09-21T03:54:00.707 INFO:teuthology.task.kernel:config {'host.a': {'kdb': 1, 'sha1': 'distro'}, 'host.b': {'kdb': 1, 'sha1': 'distro'}}, timeout 300 2024-09-21T03:54:00.707 DEBUG:teuthology.orchestra.run.smithi089:> test -f /run/.containerenv -o -f /.dockerenv 2024-09-21T03:54:00.708 DEBUG:teuthology.orchestra.run.smithi195:> test -f /run/.containerenv -o -f /.dockerenv 2024-09-21T03:54:00.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T03:54:00.725 DEBUG:teuthology.orchestra.run.smithi089:> uname -r 2024-09-21T03:54:00.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T03:54:00.726 DEBUG:teuthology.orchestra.run.smithi195:> uname -r 2024-09-21T03:54:00.783 INFO:teuthology.orchestra.run.smithi089.stdout:5.14.0-503.el9.x86_64 2024-09-21T03:54:00.784 INFO:teuthology.task.kernel:Running kernel on smithi089: 5.14.0-503.el9.x86_64 2024-09-21T03:54:00.784 DEBUG:teuthology.orchestra.run.smithi089:> sudo yum install -y kernel 2024-09-21T03:54:00.785 INFO:teuthology.orchestra.run.smithi195.stdout:5.14.0-503.el9.x86_64 2024-09-21T03:54:00.785 INFO:teuthology.task.kernel:Running kernel on smithi195: 5.14.0-503.el9.x86_64 2024-09-21T03:54:00.785 DEBUG:teuthology.orchestra.run.smithi195:> sudo yum install -y kernel 2024-09-21T03:54:03.353 INFO:teuthology.orchestra.run.smithi195.stdout:CentOS Stream 9 - BaseOS 11 MB/s | 8.3 MB 00:00 2024-09-21T03:54:04.020 INFO:teuthology.orchestra.run.smithi089.stdout:CentOS Stream 9 - BaseOS 5.9 MB/s | 8.3 MB 00:01 2024-09-21T03:54:05.121 INFO:teuthology.orchestra.run.smithi195.stdout:CentOS Stream 9 - AppStream 30 MB/s | 20 MB 00:00 2024-09-21T03:54:05.921 INFO:teuthology.orchestra.run.smithi089.stdout:CentOS Stream 9 - AppStream 25 MB/s | 20 MB 00:00 2024-09-21T03:54:11.346 INFO:teuthology.orchestra.run.smithi195.stdout:CentOS Stream 9 - CRB 2.8 MB/s | 6.5 MB 00:02 2024-09-21T03:54:12.371 INFO:teuthology.orchestra.run.smithi089.stdout:CentOS Stream 9 - CRB 2.4 MB/s | 6.5 MB 00:02 2024-09-21T03:54:12.713 INFO:teuthology.orchestra.run.smithi195.stdout:CentOS Stream 9 - Extras packages 133 kB/s | 19 kB 00:00 2024-09-21T03:54:13.559 INFO:teuthology.orchestra.run.smithi195.stdout:Extra Packages for Enterprise Linux 31 MB/s | 23 MB 00:00 2024-09-21T03:54:14.005 INFO:teuthology.orchestra.run.smithi089.stdout:CentOS Stream 9 - Extras packages 45 kB/s | 19 kB 00:00 2024-09-21T03:54:15.010 INFO:teuthology.orchestra.run.smithi089.stdout:Extra Packages for Enterprise Linux 26 MB/s | 23 MB 00:00 2024-09-21T03:54:19.024 INFO:teuthology.orchestra.run.smithi195.stdout:lab-extras 29 kB/s | 1.7 kB 00:00 2024-09-21T03:54:20.382 INFO:teuthology.orchestra.run.smithi089.stdout:lab-extras 32 kB/s | 1.7 kB 00:00 2024-09-21T03:54:20.946 INFO:teuthology.orchestra.run.smithi195.stdout:Package kernel-5.14.0-239.el9.x86_64 is already installed. 2024-09-21T03:54:20.947 INFO:teuthology.orchestra.run.smithi195.stdout:Package kernel-5.14.0-503.el9.x86_64 is already installed. 2024-09-21T03:54:21.042 INFO:teuthology.orchestra.run.smithi195.stdout:Dependencies resolved. 2024-09-21T03:54:21.052 INFO:teuthology.orchestra.run.smithi195.stdout:================================================================================ 2024-09-21T03:54:21.052 INFO:teuthology.orchestra.run.smithi195.stdout: Package Architecture Version Repository Size 2024-09-21T03:54:21.052 INFO:teuthology.orchestra.run.smithi195.stdout:================================================================================ 2024-09-21T03:54:21.052 INFO:teuthology.orchestra.run.smithi195.stdout:Installing: 2024-09-21T03:54:21.053 INFO:teuthology.orchestra.run.smithi195.stdout: kernel x86_64 5.14.0-508.el9 baseos 36 k 2024-09-21T03:54:21.053 INFO:teuthology.orchestra.run.smithi195.stdout:Installing dependencies: 2024-09-21T03:54:21.053 INFO:teuthology.orchestra.run.smithi195.stdout: kernel-core x86_64 5.14.0-508.el9 baseos 16 M 2024-09-21T03:54:21.053 INFO:teuthology.orchestra.run.smithi195.stdout: kernel-modules x86_64 5.14.0-508.el9 baseos 35 M 2024-09-21T03:54:21.053 INFO:teuthology.orchestra.run.smithi195.stdout: kernel-modules-core x86_64 5.14.0-508.el9 baseos 29 M 2024-09-21T03:54:21.053 INFO:teuthology.orchestra.run.smithi195.stdout: 2024-09-21T03:54:21.053 INFO:teuthology.orchestra.run.smithi195.stdout:Transaction Summary 2024-09-21T03:54:21.053 INFO:teuthology.orchestra.run.smithi195.stdout:================================================================================ 2024-09-21T03:54:21.053 INFO:teuthology.orchestra.run.smithi195.stdout:Install 4 Packages 2024-09-21T03:54:21.053 INFO:teuthology.orchestra.run.smithi195.stdout: 2024-09-21T03:54:21.054 INFO:teuthology.orchestra.run.smithi195.stdout:Total download size: 79 M 2024-09-21T03:54:21.054 INFO:teuthology.orchestra.run.smithi195.stdout:Installed size: 126 M 2024-09-21T03:54:21.054 INFO:teuthology.orchestra.run.smithi195.stdout:Downloading Packages: 2024-09-21T03:54:21.233 INFO:teuthology.orchestra.run.smithi195.stdout:(1/4): kernel-5.14.0-508.el9.x86_64.rpm 443 kB/s | 36 kB 00:00 2024-09-21T03:54:22.141 INFO:teuthology.orchestra.run.smithi195.stdout:(2/4): kernel-core-5.14.0-508.el9.x86_64.rpm 16 MB/s | 16 MB 00:00 2024-09-21T03:54:22.288 INFO:teuthology.orchestra.run.smithi089.stdout:Package kernel-5.14.0-239.el9.x86_64 is already installed. 2024-09-21T03:54:22.288 INFO:teuthology.orchestra.run.smithi089.stdout:Package kernel-5.14.0-503.el9.x86_64 is already installed. 2024-09-21T03:54:22.383 INFO:teuthology.orchestra.run.smithi089.stdout:Dependencies resolved. 2024-09-21T03:54:22.392 INFO:teuthology.orchestra.run.smithi089.stdout:================================================================================ 2024-09-21T03:54:22.392 INFO:teuthology.orchestra.run.smithi089.stdout: Package Architecture Version Repository Size 2024-09-21T03:54:22.392 INFO:teuthology.orchestra.run.smithi089.stdout:================================================================================ 2024-09-21T03:54:22.392 INFO:teuthology.orchestra.run.smithi089.stdout:Installing: 2024-09-21T03:54:22.392 INFO:teuthology.orchestra.run.smithi089.stdout: kernel x86_64 5.14.0-508.el9 baseos 36 k 2024-09-21T03:54:22.392 INFO:teuthology.orchestra.run.smithi089.stdout:Installing dependencies: 2024-09-21T03:54:22.392 INFO:teuthology.orchestra.run.smithi089.stdout: kernel-core x86_64 5.14.0-508.el9 baseos 16 M 2024-09-21T03:54:22.392 INFO:teuthology.orchestra.run.smithi089.stdout: kernel-modules x86_64 5.14.0-508.el9 baseos 35 M 2024-09-21T03:54:22.392 INFO:teuthology.orchestra.run.smithi089.stdout: kernel-modules-core x86_64 5.14.0-508.el9 baseos 29 M 2024-09-21T03:54:22.392 INFO:teuthology.orchestra.run.smithi089.stdout: 2024-09-21T03:54:22.392 INFO:teuthology.orchestra.run.smithi089.stdout:Transaction Summary 2024-09-21T03:54:22.392 INFO:teuthology.orchestra.run.smithi089.stdout:================================================================================ 2024-09-21T03:54:22.393 INFO:teuthology.orchestra.run.smithi089.stdout:Install 4 Packages 2024-09-21T03:54:22.393 INFO:teuthology.orchestra.run.smithi089.stdout: 2024-09-21T03:54:22.393 INFO:teuthology.orchestra.run.smithi089.stdout:Total download size: 79 M 2024-09-21T03:54:22.393 INFO:teuthology.orchestra.run.smithi089.stdout:Installed size: 126 M 2024-09-21T03:54:22.393 INFO:teuthology.orchestra.run.smithi089.stdout:Downloading Packages: 2024-09-21T03:54:22.400 INFO:teuthology.orchestra.run.smithi195.stdout:(3/4): kernel-modules-core-5.14.0-508.el9.x86_6 24 MB/s | 29 MB 00:01 2024-09-21T03:54:22.616 INFO:teuthology.orchestra.run.smithi089.stdout:(1/4): kernel-5.14.0-508.el9.x86_64.rpm 351 kB/s | 36 kB 00:00 2024-09-21T03:54:22.742 INFO:teuthology.orchestra.run.smithi195.stdout:(4/4): kernel-modules-5.14.0-508.el9.x86_64.rpm 22 MB/s | 35 MB 00:01 2024-09-21T03:54:22.742 INFO:teuthology.orchestra.run.smithi195.stdout:-------------------------------------------------------------------------------- 2024-09-21T03:54:22.742 INFO:teuthology.orchestra.run.smithi195.stdout:Total 47 MB/s | 79 MB 00:01 2024-09-21T03:54:22.992 INFO:teuthology.orchestra.run.smithi089.stdout:(2/4): kernel-core-5.14.0-508.el9.x86_64.rpm 33 MB/s | 16 MB 00:00 2024-09-21T03:54:23.392 INFO:teuthology.orchestra.run.smithi089.stdout:(3/4): kernel-modules-core-5.14.0-508.el9.x86_6 37 MB/s | 29 MB 00:00 2024-09-21T03:54:23.426 INFO:teuthology.orchestra.run.smithi195.stdout:Running transaction check 2024-09-21T03:54:23.717 INFO:teuthology.orchestra.run.smithi089.stdout:(4/4): kernel-modules-5.14.0-508.el9.x86_64.rpm 29 MB/s | 35 MB 00:01 2024-09-21T03:54:23.717 INFO:teuthology.orchestra.run.smithi089.stdout:-------------------------------------------------------------------------------- 2024-09-21T03:54:23.718 INFO:teuthology.orchestra.run.smithi089.stdout:Total 59 MB/s | 79 MB 00:01 2024-09-21T03:54:23.755 INFO:teuthology.orchestra.run.smithi195.stdout:Transaction check succeeded. 2024-09-21T03:54:23.755 INFO:teuthology.orchestra.run.smithi195.stdout:Running transaction test 2024-09-21T03:54:24.399 INFO:teuthology.orchestra.run.smithi089.stdout:Running transaction check 2024-09-21T03:54:24.723 INFO:teuthology.orchestra.run.smithi195.stdout:Transaction test succeeded. 2024-09-21T03:54:24.723 INFO:teuthology.orchestra.run.smithi195.stdout:Running transaction 2024-09-21T03:54:24.731 INFO:teuthology.orchestra.run.smithi089.stdout:Transaction check succeeded. 2024-09-21T03:54:24.731 INFO:teuthology.orchestra.run.smithi089.stdout:Running transaction test 2024-09-21T03:54:25.688 INFO:teuthology.orchestra.run.smithi089.stdout:Transaction test succeeded. 2024-09-21T03:54:25.688 INFO:teuthology.orchestra.run.smithi089.stdout:Running transaction 2024-09-21T03:54:27.128 INFO:teuthology.orchestra.run.smithi195.stdout: Preparing : 1/1 2024-09-21T03:54:27.767 INFO:teuthology.orchestra.run.smithi195.stdout: Installing : kernel-modules-core-5.14.0-508.el9.x86_64 1/4 2024-09-21T03:54:27.851 INFO:teuthology.orchestra.run.smithi195.stdout: Installing : kernel-core-5.14.0-508.el9.x86_64 2/4 2024-09-21T03:54:28.104 INFO:teuthology.orchestra.run.smithi089.stdout: Preparing : 1/1 2024-09-21T03:54:28.745 INFO:teuthology.orchestra.run.smithi089.stdout: Installing : kernel-modules-core-5.14.0-508.el9.x86_64 1/4 2024-09-21T03:54:28.826 INFO:teuthology.orchestra.run.smithi089.stdout: Installing : kernel-core-5.14.0-508.el9.x86_64 2/4 2024-09-21T03:54:30.605 INFO:teuthology.orchestra.run.smithi195.stdout: Running scriptlet: kernel-core-5.14.0-508.el9.x86_64 2/4 2024-09-21T03:54:30.730 INFO:teuthology.orchestra.run.smithi195.stdout: Installing : kernel-modules-5.14.0-508.el9.x86_64 3/4 2024-09-21T03:54:31.572 INFO:teuthology.orchestra.run.smithi089.stdout: Running scriptlet: kernel-core-5.14.0-508.el9.x86_64 2/4 2024-09-21T03:54:31.705 INFO:teuthology.orchestra.run.smithi089.stdout: Installing : kernel-modules-5.14.0-508.el9.x86_64 3/4 2024-09-21T03:54:35.566 INFO:teuthology.orchestra.run.smithi195.stdout: Running scriptlet: kernel-modules-5.14.0-508.el9.x86_64 3/4 2024-09-21T03:54:35.605 INFO:teuthology.orchestra.run.smithi195.stdout: Installing : kernel-5.14.0-508.el9.x86_64 4/4 2024-09-21T03:54:36.530 INFO:teuthology.orchestra.run.smithi089.stdout: Running scriptlet: kernel-modules-5.14.0-508.el9.x86_64 3/4 2024-09-21T03:54:36.572 INFO:teuthology.orchestra.run.smithi089.stdout: Installing : kernel-5.14.0-508.el9.x86_64 4/4 2024-09-21T03:54:40.426 INFO:teuthology.orchestra.run.smithi195.stdout: Running scriptlet: kernel-modules-core-5.14.0-508.el9.x86_64 4/4 2024-09-21T03:54:41.407 INFO:teuthology.orchestra.run.smithi089.stdout: Running scriptlet: kernel-modules-core-5.14.0-508.el9.x86_64 4/4 2024-09-21T03:55:18.254 INFO:teuthology.orchestra.run.smithi195.stdout: Running scriptlet: kernel-core-5.14.0-508.el9.x86_64 4/4 2024-09-21T03:55:18.294 INFO:teuthology.orchestra.run.smithi195.stdout: Running scriptlet: kernel-modules-5.14.0-508.el9.x86_64 4/4 2024-09-21T03:55:18.613 INFO:teuthology.orchestra.run.smithi195.stdout: Running scriptlet: kernel-5.14.0-508.el9.x86_64 4/4 2024-09-21T03:55:18.613 INFO:teuthology.orchestra.run.smithi195.stdout: Verifying : kernel-5.14.0-508.el9.x86_64 1/4 2024-09-21T03:55:18.613 INFO:teuthology.orchestra.run.smithi195.stdout: Verifying : kernel-core-5.14.0-508.el9.x86_64 2/4 2024-09-21T03:55:18.613 INFO:teuthology.orchestra.run.smithi195.stdout: Verifying : kernel-modules-5.14.0-508.el9.x86_64 3/4 2024-09-21T03:55:19.432 INFO:teuthology.orchestra.run.smithi089.stdout: Running scriptlet: kernel-core-5.14.0-508.el9.x86_64 4/4 2024-09-21T03:55:19.438 INFO:teuthology.orchestra.run.smithi195.stdout: Verifying : kernel-modules-core-5.14.0-508.el9.x86_64 4/4 2024-09-21T03:55:19.438 INFO:teuthology.orchestra.run.smithi195.stdout: 2024-09-21T03:55:19.438 INFO:teuthology.orchestra.run.smithi195.stdout:Installed: 2024-09-21T03:55:19.438 INFO:teuthology.orchestra.run.smithi195.stdout: kernel-5.14.0-508.el9.x86_64 2024-09-21T03:55:19.438 INFO:teuthology.orchestra.run.smithi195.stdout: kernel-core-5.14.0-508.el9.x86_64 2024-09-21T03:55:19.438 INFO:teuthology.orchestra.run.smithi195.stdout: kernel-modules-5.14.0-508.el9.x86_64 2024-09-21T03:55:19.438 INFO:teuthology.orchestra.run.smithi195.stdout: kernel-modules-core-5.14.0-508.el9.x86_64 2024-09-21T03:55:19.439 INFO:teuthology.orchestra.run.smithi195.stdout: 2024-09-21T03:55:19.439 INFO:teuthology.orchestra.run.smithi195.stdout:Complete! 2024-09-21T03:55:19.478 INFO:teuthology.orchestra.run.smithi089.stdout: Running scriptlet: kernel-modules-5.14.0-508.el9.x86_64 4/4 2024-09-21T03:55:19.750 INFO:teuthology.orchestra.run.smithi089.stdout: Running scriptlet: kernel-5.14.0-508.el9.x86_64 4/4 2024-09-21T03:55:19.750 INFO:teuthology.orchestra.run.smithi089.stdout: Verifying : kernel-5.14.0-508.el9.x86_64 1/4 2024-09-21T03:55:19.750 INFO:teuthology.orchestra.run.smithi089.stdout: Verifying : kernel-core-5.14.0-508.el9.x86_64 2/4 2024-09-21T03:55:19.750 INFO:teuthology.orchestra.run.smithi089.stdout: Verifying : kernel-modules-5.14.0-508.el9.x86_64 3/4 2024-09-21T03:55:19.779 DEBUG:teuthology.orchestra.run.smithi195:> rpm -q kernel | sort -rV | head -n 1 2024-09-21T03:55:19.863 INFO:teuthology.orchestra.run.smithi195.stdout:kernel-5.14.0-508.el9.x86_64 2024-09-21T03:55:19.864 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-508.el9.x86_64 2024-09-21T03:55:19.864 INFO:teuthology.task.kernel:Not newest distro kernel. Current: 5.14.0-503.el9.x86_64 Expected: 5.14.0-508.el9.x86_64 2024-09-21T03:55:19.864 INFO:teuthology.task.kernel:Skipping firmware on distro kernel 2024-09-21T03:55:19.864 DEBUG:teuthology.task.kernel:src is distro, skipping download 2024-09-21T03:55:19.864 INFO:teuthology.task.kernel:Installing distro kernel on host.b... 2024-09-21T03:55:19.864 DEBUG:teuthology.task.kernel:install_kernel(remote=ubuntu@smithi195.front.sepia.ceph.com, path=None, version=distro) 2024-09-21T03:55:19.864 DEBUG:teuthology.orchestra.run.smithi195:> rpm -q kernel | sort -rV | head -n 1 2024-09-21T03:55:19.923 INFO:teuthology.orchestra.run.smithi195.stdout:kernel-5.14.0-508.el9.x86_64 2024-09-21T03:55:19.923 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-508.el9.x86_64 2024-09-21T03:55:19.923 DEBUG:teuthology.orchestra.run.smithi195:> sudo rpm -qi grub2-tools 2024-09-21T03:55:19.958 INFO:teuthology.orchestra.run.smithi195.stdout:Name : grub2-tools 2024-09-21T03:55:19.958 INFO:teuthology.orchestra.run.smithi195.stdout:Epoch : 1 2024-09-21T03:55:19.958 INFO:teuthology.orchestra.run.smithi195.stdout:Version : 2.06 2024-09-21T03:55:19.958 INFO:teuthology.orchestra.run.smithi195.stdout:Release : 82.el9 2024-09-21T03:55:19.958 INFO:teuthology.orchestra.run.smithi195.stdout:Architecture: x86_64 2024-09-21T03:55:19.958 INFO:teuthology.orchestra.run.smithi195.stdout:Install Date: Wed 28 Aug 2024 09:06:27 PM UTC 2024-09-21T03:55:19.958 INFO:teuthology.orchestra.run.smithi195.stdout:Group : Unspecified 2024-09-21T03:55:19.958 INFO:teuthology.orchestra.run.smithi195.stdout:Size : 8274670 2024-09-21T03:55:19.959 INFO:teuthology.orchestra.run.smithi195.stdout:License : GPLv3+ 2024-09-21T03:55:19.959 INFO:teuthology.orchestra.run.smithi195.stdout:Signature : RSA/SHA256, Mon 01 Jul 2024 07:46:42 AM UTC, Key ID 05b555b38483c65d 2024-09-21T03:55:19.959 INFO:teuthology.orchestra.run.smithi195.stdout:Source RPM : grub2-2.06-82.el9.src.rpm 2024-09-21T03:55:19.959 INFO:teuthology.orchestra.run.smithi195.stdout:Build Date : Fri 28 Jun 2024 09:24:44 AM UTC 2024-09-21T03:55:19.959 INFO:teuthology.orchestra.run.smithi195.stdout:Build Host : x86-05.stream.rdu2.redhat.com 2024-09-21T03:55:19.959 INFO:teuthology.orchestra.run.smithi195.stdout:Packager : builder@centos.org 2024-09-21T03:55:19.959 INFO:teuthology.orchestra.run.smithi195.stdout:Vendor : CentOS 2024-09-21T03:55:19.959 INFO:teuthology.orchestra.run.smithi195.stdout:URL : http://www.gnu.org/software/grub/ 2024-09-21T03:55:19.959 INFO:teuthology.orchestra.run.smithi195.stdout:Summary : Support tools for GRUB. 2024-09-21T03:55:19.959 INFO:teuthology.orchestra.run.smithi195.stdout:Description : 2024-09-21T03:55:19.959 INFO:teuthology.orchestra.run.smithi195.stdout: 2024-09-21T03:55:19.959 INFO:teuthology.orchestra.run.smithi195.stdout:The GRand Unified Bootloader (GRUB) is a highly configurable and 2024-09-21T03:55:19.959 INFO:teuthology.orchestra.run.smithi195.stdout:customizable bootloader with modular architecture. It supports a rich 2024-09-21T03:55:19.959 INFO:teuthology.orchestra.run.smithi195.stdout:variety of kernel formats, file systems, computer architectures and 2024-09-21T03:55:19.959 INFO:teuthology.orchestra.run.smithi195.stdout:hardware devices. 2024-09-21T03:55:19.960 INFO:teuthology.orchestra.run.smithi195.stdout: 2024-09-21T03:55:19.960 INFO:teuthology.orchestra.run.smithi195.stdout:This subpackage provides tools for support of all platforms. 2024-09-21T03:55:19.960 INFO:teuthology.task.kernel:Updating Grub Version: grub2 2024-09-21T03:55:19.960 INFO:teuthology.task.kernel:Updating grub on smithi195 to boot 5.14.0-508.el9.x86_64 2024-09-21T03:55:19.961 DEBUG:teuthology.orchestra.run.smithi195:> sudo grub2-mkconfig -o /boot/grub2/grub.cfg 2024-09-21T03:55:20.577 INFO:teuthology.orchestra.run.smithi089.stdout: Verifying : kernel-modules-core-5.14.0-508.el9.x86_64 4/4 2024-09-21T03:55:20.577 INFO:teuthology.orchestra.run.smithi089.stdout: 2024-09-21T03:55:20.577 INFO:teuthology.orchestra.run.smithi089.stdout:Installed: 2024-09-21T03:55:20.577 INFO:teuthology.orchestra.run.smithi089.stdout: kernel-5.14.0-508.el9.x86_64 2024-09-21T03:55:20.577 INFO:teuthology.orchestra.run.smithi089.stdout: kernel-core-5.14.0-508.el9.x86_64 2024-09-21T03:55:20.578 INFO:teuthology.orchestra.run.smithi089.stdout: kernel-modules-5.14.0-508.el9.x86_64 2024-09-21T03:55:20.578 INFO:teuthology.orchestra.run.smithi089.stdout: kernel-modules-core-5.14.0-508.el9.x86_64 2024-09-21T03:55:20.578 INFO:teuthology.orchestra.run.smithi089.stdout: 2024-09-21T03:55:20.578 INFO:teuthology.orchestra.run.smithi089.stdout:Complete! 2024-09-21T03:55:20.621 INFO:teuthology.orchestra.run.smithi195.stderr:Generating grub configuration file ... 2024-09-21T03:55:20.959 DEBUG:teuthology.orchestra.run.smithi089:> rpm -q kernel | sort -rV | head -n 1 2024-09-21T03:55:21.052 INFO:teuthology.orchestra.run.smithi089.stdout:kernel-5.14.0-508.el9.x86_64 2024-09-21T03:55:21.052 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-508.el9.x86_64 2024-09-21T03:55:21.052 INFO:teuthology.task.kernel:Not newest distro kernel. Current: 5.14.0-503.el9.x86_64 Expected: 5.14.0-508.el9.x86_64 2024-09-21T03:55:21.052 INFO:teuthology.task.kernel:Skipping firmware on distro kernel 2024-09-21T03:55:21.052 DEBUG:teuthology.task.kernel:src is distro, skipping download 2024-09-21T03:55:21.053 INFO:teuthology.task.kernel:Installing distro kernel on host.a... 2024-09-21T03:55:21.053 DEBUG:teuthology.task.kernel:install_kernel(remote=ubuntu@smithi089.front.sepia.ceph.com, path=None, version=distro) 2024-09-21T03:55:21.053 DEBUG:teuthology.orchestra.run.smithi089:> rpm -q kernel | sort -rV | head -n 1 2024-09-21T03:55:21.110 INFO:teuthology.orchestra.run.smithi089.stdout:kernel-5.14.0-508.el9.x86_64 2024-09-21T03:55:21.111 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-508.el9.x86_64 2024-09-21T03:55:21.111 DEBUG:teuthology.orchestra.run.smithi089:> sudo rpm -qi grub2-tools 2024-09-21T03:55:21.142 INFO:teuthology.orchestra.run.smithi089.stdout:Name : grub2-tools 2024-09-21T03:55:21.142 INFO:teuthology.orchestra.run.smithi089.stdout:Epoch : 1 2024-09-21T03:55:21.142 INFO:teuthology.orchestra.run.smithi089.stdout:Version : 2.06 2024-09-21T03:55:21.142 INFO:teuthology.orchestra.run.smithi089.stdout:Release : 82.el9 2024-09-21T03:55:21.142 INFO:teuthology.orchestra.run.smithi089.stdout:Architecture: x86_64 2024-09-21T03:55:21.142 INFO:teuthology.orchestra.run.smithi089.stdout:Install Date: Wed 28 Aug 2024 09:06:27 PM UTC 2024-09-21T03:55:21.142 INFO:teuthology.orchestra.run.smithi089.stdout:Group : Unspecified 2024-09-21T03:55:21.143 INFO:teuthology.orchestra.run.smithi089.stdout:Size : 8274670 2024-09-21T03:55:21.143 INFO:teuthology.orchestra.run.smithi089.stdout:License : GPLv3+ 2024-09-21T03:55:21.143 INFO:teuthology.orchestra.run.smithi089.stdout:Signature : RSA/SHA256, Mon 01 Jul 2024 07:46:42 AM UTC, Key ID 05b555b38483c65d 2024-09-21T03:55:21.143 INFO:teuthology.orchestra.run.smithi089.stdout:Source RPM : grub2-2.06-82.el9.src.rpm 2024-09-21T03:55:21.143 INFO:teuthology.orchestra.run.smithi089.stdout:Build Date : Fri 28 Jun 2024 09:24:44 AM UTC 2024-09-21T03:55:21.143 INFO:teuthology.orchestra.run.smithi089.stdout:Build Host : x86-05.stream.rdu2.redhat.com 2024-09-21T03:55:21.143 INFO:teuthology.orchestra.run.smithi089.stdout:Packager : builder@centos.org 2024-09-21T03:55:21.143 INFO:teuthology.orchestra.run.smithi089.stdout:Vendor : CentOS 2024-09-21T03:55:21.143 INFO:teuthology.orchestra.run.smithi089.stdout:URL : http://www.gnu.org/software/grub/ 2024-09-21T03:55:21.143 INFO:teuthology.orchestra.run.smithi089.stdout:Summary : Support tools for GRUB. 2024-09-21T03:55:21.143 INFO:teuthology.orchestra.run.smithi089.stdout:Description : 2024-09-21T03:55:21.143 INFO:teuthology.orchestra.run.smithi089.stdout: 2024-09-21T03:55:21.143 INFO:teuthology.orchestra.run.smithi089.stdout:The GRand Unified Bootloader (GRUB) is a highly configurable and 2024-09-21T03:55:21.143 INFO:teuthology.orchestra.run.smithi089.stdout:customizable bootloader with modular architecture. It supports a rich 2024-09-21T03:55:21.144 INFO:teuthology.orchestra.run.smithi089.stdout:variety of kernel formats, file systems, computer architectures and 2024-09-21T03:55:21.144 INFO:teuthology.orchestra.run.smithi089.stdout:hardware devices. 2024-09-21T03:55:21.144 INFO:teuthology.orchestra.run.smithi089.stdout: 2024-09-21T03:55:21.144 INFO:teuthology.orchestra.run.smithi089.stdout:This subpackage provides tools for support of all platforms. 2024-09-21T03:55:21.144 INFO:teuthology.task.kernel:Updating Grub Version: grub2 2024-09-21T03:55:21.145 INFO:teuthology.task.kernel:Updating grub on smithi089 to boot 5.14.0-508.el9.x86_64 2024-09-21T03:55:21.145 DEBUG:teuthology.orchestra.run.smithi089:> sudo grub2-mkconfig -o /boot/grub2/grub.cfg 2024-09-21T03:55:21.774 INFO:teuthology.orchestra.run.smithi089.stderr:Generating grub configuration file ... 2024-09-21T03:55:21.938 INFO:teuthology.orchestra.run.smithi195.stderr:Adding boot menu entry for UEFI Firmware Settings ... 2024-09-21T03:55:21.965 INFO:teuthology.orchestra.run.smithi195.stderr:done 2024-09-21T03:55:21.967 DEBUG:teuthology.orchestra.run.smithi195:> mktemp 2024-09-21T03:55:21.982 INFO:teuthology.orchestra.run.smithi195.stdout:/tmp/tmp.9mSQsVdTwH 2024-09-21T03:55:21.982 DEBUG:teuthology.orchestra.run.smithi195:> sudo cp /boot/grub2/grub.cfg /tmp/tmp.9mSQsVdTwH 2024-09-21T03:55:22.058 DEBUG:teuthology.orchestra.run.smithi195:> sudo chmod 0666 /tmp/tmp.9mSQsVdTwH 2024-09-21T03:55:22.199 DEBUG:teuthology.orchestra.remote:smithi195:/tmp/tmp.9mSQsVdTwH is 6KB 2024-09-21T03:55:22.247 DEBUG:teuthology.orchestra.run.smithi195:> rm -fr /tmp/tmp.9mSQsVdTwH 2024-09-21T03:55:22.261 DEBUG:teuthology.orchestra.run.smithi195:> sudo /bin/ls /boot/loader/entries || true 2024-09-21T03:55:22.327 INFO:teuthology.orchestra.run.smithi195.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-0-rescue.conf 2024-09-21T03:55:22.327 INFO:teuthology.orchestra.run.smithi195.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-239.el9.x86_64.conf 2024-09-21T03:55:22.328 INFO:teuthology.orchestra.run.smithi195.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-503.el9.x86_64.conf 2024-09-21T03:55:22.328 INFO:teuthology.orchestra.run.smithi195.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-508.el9.x86_64.conf 2024-09-21T03:55:22.329 DEBUG:teuthology.orchestra.run.smithi195:> sudo grub2-set-default fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-508.el9.x86_64 2024-09-21T03:55:22.481 DEBUG:teuthology.orchestra.run.smithi195:> sudo shutdown -r now 2024-09-21T03:55:23.091 INFO:teuthology.orchestra.run.smithi089.stderr:Adding boot menu entry for UEFI Firmware Settings ... 2024-09-21T03:55:23.124 INFO:teuthology.orchestra.run.smithi089.stderr:done 2024-09-21T03:55:23.126 DEBUG:teuthology.orchestra.run.smithi089:> mktemp 2024-09-21T03:55:23.141 INFO:teuthology.orchestra.run.smithi089.stdout:/tmp/tmp.laW7LKjJ1P 2024-09-21T03:55:23.142 DEBUG:teuthology.orchestra.run.smithi089:> sudo cp /boot/grub2/grub.cfg /tmp/tmp.laW7LKjJ1P 2024-09-21T03:55:23.208 DEBUG:teuthology.orchestra.run.smithi089:> sudo chmod 0666 /tmp/tmp.laW7LKjJ1P 2024-09-21T03:55:23.342 DEBUG:teuthology.orchestra.remote:smithi089:/tmp/tmp.laW7LKjJ1P is 6KB 2024-09-21T03:55:23.390 DEBUG:teuthology.orchestra.run.smithi089:> rm -fr /tmp/tmp.laW7LKjJ1P 2024-09-21T03:55:23.416 DEBUG:teuthology.orchestra.run.smithi089:> sudo /bin/ls /boot/loader/entries || true 2024-09-21T03:55:23.467 INFO:teuthology.orchestra.run.smithi089.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-0-rescue.conf 2024-09-21T03:55:23.468 INFO:teuthology.orchestra.run.smithi089.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-239.el9.x86_64.conf 2024-09-21T03:55:23.468 INFO:teuthology.orchestra.run.smithi089.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-503.el9.x86_64.conf 2024-09-21T03:55:23.468 INFO:teuthology.orchestra.run.smithi089.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-508.el9.x86_64.conf 2024-09-21T03:55:23.469 DEBUG:teuthology.orchestra.run.smithi089:> sudo grub2-set-default fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-508.el9.x86_64 2024-09-21T03:55:23.622 DEBUG:teuthology.orchestra.run.smithi089:> sudo shutdown -r now 2024-09-21T03:55:52.496 INFO:teuthology.task.kernel:Checking client host.b for new kernel version... 2024-09-21T03:55:52.496 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi195.front.sepia.ceph.com' 2024-09-21T03:55:52.497 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi195.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-21T03:55:53.626 INFO:teuthology.task.kernel:Checking client host.a for new kernel version... 2024-09-21T03:55:53.627 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi089.front.sepia.ceph.com' 2024-09-21T03:55:53.627 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi089.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-21T03:56:12.143 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.89 2024-09-21T03:56:21.150 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi089.front.sepia.ceph.com' 2024-09-21T03:56:21.151 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi089.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-21T03:56:24.207 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.89 2024-09-21T03:56:36.218 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi089.front.sepia.ceph.com' 2024-09-21T03:56:36.219 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi089.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-21T03:56:52.515 DEBUG:teuthology.orchestra.remote:timed out 2024-09-21T03:56:54.639 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.89 2024-09-21T03:57:01.522 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi195.front.sepia.ceph.com' 2024-09-21T03:57:01.523 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi195.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-21T03:57:01.900 DEBUG:teuthology.orchestra.run.smithi195:> true 2024-09-21T03:57:02.296 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi195.front.sepia.ceph.com' 2024-09-21T03:57:02.296 INFO:teuthology.task.kernel:Checking kernel version of host.b, want "5.14.0-508.el9.x86_64"... 2024-09-21T03:57:02.296 DEBUG:teuthology.orchestra.run.smithi195:> uname -r 2024-09-21T03:57:02.311 INFO:teuthology.orchestra.run.smithi195.stdout:5.14.0-508.el9.x86_64 2024-09-21T03:57:02.311 DEBUG:teuthology.task.kernel:current kernel version is 5.14.0-508.el9.x86_64 vs 5.14.0-508.el9.x86_64 2024-09-21T03:57:02.311 DEBUG:teuthology.task.kernel:utsrelease strings match, do not need to install 2024-09-21T03:57:02.311 DEBUG:teuthology.task.kernel:Distro of this test job: centos 2024-09-21T03:57:03.312 INFO:teuthology.task.kernel:Enabling kdb on host.b... 2024-09-21T03:57:03.312 DEBUG:teuthology.orchestra.run.smithi195:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2024-09-21T03:57:03.434 INFO:teuthology.orchestra.run.smithi195.stdout:ttyS1 2024-09-21T03:57:03.472 DEBUG:teuthology.parallel:result is None 2024-09-21T03:57:09.639 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi089.front.sepia.ceph.com' 2024-09-21T03:57:09.640 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi089.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-21T03:57:09.961 DEBUG:teuthology.orchestra.run.smithi089:> true 2024-09-21T03:57:10.403 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi089.front.sepia.ceph.com' 2024-09-21T03:57:10.403 INFO:teuthology.task.kernel:Checking kernel version of host.a, want "5.14.0-508.el9.x86_64"... 2024-09-21T03:57:10.404 DEBUG:teuthology.orchestra.run.smithi089:> uname -r 2024-09-21T03:57:10.462 INFO:teuthology.orchestra.run.smithi089.stdout:5.14.0-508.el9.x86_64 2024-09-21T03:57:10.462 DEBUG:teuthology.task.kernel:current kernel version is 5.14.0-508.el9.x86_64 vs 5.14.0-508.el9.x86_64 2024-09-21T03:57:10.463 DEBUG:teuthology.task.kernel:utsrelease strings match, do not need to install 2024-09-21T03:57:10.463 DEBUG:teuthology.task.kernel:Distro of this test job: centos 2024-09-21T03:57:11.463 INFO:teuthology.task.kernel:Enabling kdb on host.a... 2024-09-21T03:57:11.464 DEBUG:teuthology.orchestra.run.smithi089:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2024-09-21T03:57:11.597 INFO:teuthology.orchestra.run.smithi089.stdout:ttyS1 2024-09-21T03:57:11.630 DEBUG:teuthology.parallel:result is None 2024-09-21T03:57:11.631 INFO:teuthology.run_tasks:Running task internal.base... 2024-09-21T03:57:11.637 INFO:teuthology.task.internal:Creating test directory... 2024-09-21T03:57:11.637 DEBUG:teuthology.orchestra.run.smithi089:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2024-09-21T03:57:11.639 DEBUG:teuthology.orchestra.run.smithi195:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2024-09-21T03:57:11.689 INFO:teuthology.run_tasks:Running task internal.archive_upload... 2024-09-21T03:57:11.695 INFO:teuthology.run_tasks:Running task internal.archive... 2024-09-21T03:57:11.704 INFO:teuthology.task.internal:Creating archive directory... 2024-09-21T03:57:11.704 DEBUG:teuthology.orchestra.run.smithi089:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2024-09-21T03:57:11.707 DEBUG:teuthology.orchestra.run.smithi195:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2024-09-21T03:57:11.812 INFO:teuthology.run_tasks:Running task internal.coredump... 2024-09-21T03:57:11.818 INFO:teuthology.task.internal:Enabling coredump saving... 2024-09-21T03:57:11.818 DEBUG:teuthology.orchestra.run.smithi089:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2024-09-21T03:57:11.821 DEBUG:teuthology.orchestra.run.smithi195:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2024-09-21T03:57:11.852 INFO:teuthology.orchestra.run.smithi195.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-09-21T03:57:11.863 INFO:teuthology.orchestra.run.smithi195.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-09-21T03:57:11.887 INFO:teuthology.orchestra.run.smithi089.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-09-21T03:57:11.907 INFO:teuthology.orchestra.run.smithi089.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-09-21T03:57:11.909 INFO:teuthology.run_tasks:Running task internal.sudo... 2024-09-21T03:57:11.915 INFO:teuthology.task.internal:Configuring sudo... 2024-09-21T03:57:11.915 DEBUG:teuthology.orchestra.run.smithi089:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2024-09-21T03:57:11.918 DEBUG:teuthology.orchestra.run.smithi195:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2024-09-21T03:57:11.967 INFO:teuthology.run_tasks:Running task internal.syslog... 2024-09-21T03:57:11.976 INFO:teuthology.task.internal.syslog:Starting syslog monitoring... 2024-09-21T03:57:11.976 DEBUG:teuthology.orchestra.run.smithi089:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2024-09-21T03:57:11.978 DEBUG:teuthology.orchestra.run.smithi195:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2024-09-21T03:57:12.012 DEBUG:teuthology.orchestra.run.smithi089:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2024-09-21T03:57:12.119 DEBUG:teuthology.orchestra.run.smithi089:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2024-09-21T03:57:12.172 DEBUG:teuthology.orchestra.run.smithi089:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2024-09-21T03:57:12.258 DEBUG:teuthology.orchestra.run.smithi089:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2024-09-21T03:57:12.303 DEBUG:teuthology.orchestra.run.smithi089:> set -ex 2024-09-21T03:57:12.304 DEBUG:teuthology.orchestra.run.smithi089:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2024-09-21T03:57:12.372 DEBUG:teuthology.orchestra.run.smithi195:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2024-09-21T03:57:12.416 DEBUG:teuthology.orchestra.run.smithi195:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2024-09-21T03:57:12.466 DEBUG:teuthology.orchestra.run.smithi195:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2024-09-21T03:57:12.541 DEBUG:teuthology.orchestra.run.smithi195:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2024-09-21T03:57:12.594 DEBUG:teuthology.orchestra.run.smithi195:> set -ex 2024-09-21T03:57:12.595 DEBUG:teuthology.orchestra.run.smithi195:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2024-09-21T03:57:12.662 DEBUG:teuthology.orchestra.run.smithi089:> sudo service rsyslog restart 2024-09-21T03:57:12.664 DEBUG:teuthology.orchestra.run.smithi195:> sudo service rsyslog restart 2024-09-21T03:57:12.752 INFO:teuthology.orchestra.run.smithi195.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-09-21T03:57:12.753 INFO:teuthology.orchestra.run.smithi089.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-09-21T03:57:13.266 INFO:teuthology.run_tasks:Running task internal.timer... 2024-09-21T03:57:13.276 INFO:teuthology.task.internal:Starting timer... 2024-09-21T03:57:13.276 INFO:teuthology.run_tasks:Running task pcp... 2024-09-21T03:57:13.287 INFO:teuthology.run_tasks:Running task selinux... 2024-09-21T03:57:13.301 DEBUG:teuthology.task:Applying overrides for task selinux: {'allowlist': ['scontext=system_u:system_r:logrotate_t:s0']} 2024-09-21T03:57:13.302 DEBUG:teuthology.orchestra.run.smithi089:> sudo service auditd rotate 2024-09-21T03:57:13.344 INFO:teuthology.orchestra.run.smithi089.stdout:Rotating logs: 2024-09-21T03:57:13.346 DEBUG:teuthology.orchestra.run.smithi195:> sudo service auditd rotate 2024-09-21T03:57:13.426 INFO:teuthology.orchestra.run.smithi195.stdout:Rotating logs: 2024-09-21T03:57:13.428 DEBUG:teuthology.task.selinux:Getting current SELinux state 2024-09-21T03:57:13.428 DEBUG:teuthology.orchestra.run.smithi089:> /usr/sbin/getenforce 2024-09-21T03:57:13.449 INFO:teuthology.orchestra.run.smithi089.stdout:Permissive 2024-09-21T03:57:13.450 DEBUG:teuthology.orchestra.run.smithi195:> /usr/sbin/getenforce 2024-09-21T03:57:13.487 INFO:teuthology.orchestra.run.smithi195.stdout:Permissive 2024-09-21T03:57:13.487 DEBUG:teuthology.task.selinux:Existing SELinux modes: {'ubuntu@smithi089.front.sepia.ceph.com': 'permissive', 'ubuntu@smithi195.front.sepia.ceph.com': 'permissive'} 2024-09-21T03:57:13.487 DEBUG:teuthology.orchestra.run.smithi089:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e context=system_u:system_r:getty_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 2024-09-21T03:57:13.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T03:57:13.515 DEBUG:teuthology.orchestra.run.smithi195:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e context=system_u:system_r:getty_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 2024-09-21T03:57:13.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T03:57:13.556 INFO:teuthology.task.selinux:Putting SELinux into permissive mode 2024-09-21T03:57:13.556 DEBUG:teuthology.orchestra.run.smithi089:> sudo /usr/sbin/setenforce permissive 2024-09-21T03:57:13.597 DEBUG:teuthology.orchestra.run.smithi195:> sudo /usr/sbin/setenforce permissive 2024-09-21T03:57:13.634 INFO:teuthology.run_tasks:Running task ansible.cephlab... 2024-09-21T03:57:13.644 DEBUG:teuthology.repo_utils:Resetting repo at /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main to origin/main 2024-09-21T03:57:13.655 INFO:teuthology.task.ansible:Playbook: [{'import_playbook': 'ansible_managed.yml'}, {'import_playbook': 'teuthology.yml'}, {'hosts': 'testnodes', 'tasks': [{'set_fact': {'ran_from_cephlab_playbook': True}}]}, {'import_playbook': 'testnodes.yml'}, {'import_playbook': 'container-host.yml'}, {'import_playbook': 'cobbler.yml'}, {'import_playbook': 'paddles.yml'}, {'import_playbook': 'pulpito.yml'}, {'hosts': 'testnodes', 'become': True, 'tasks': [{'name': 'Touch /ceph-qa-ready', 'file': {'path': '/ceph-qa-ready', 'state': 'touch'}, 'when': 'ran_from_cephlab_playbook|bool'}]}] 2024-09-21T03:57:13.655 DEBUG:teuthology.task.ansible:Running ansible-playbook -v --extra-vars '{"ansible_ssh_user": "ubuntu"}' -i /etc/ansible/hosts --limit smithi089.front.sepia.ceph.com,smithi195.front.sepia.ceph.com /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main/cephlab.yml 2024-09-21T04:00:57.358 DEBUG:teuthology.task.ansible:Reconnecting to [Remote(name='ubuntu@smithi089.front.sepia.ceph.com'), Remote(name='ubuntu@smithi195.front.sepia.ceph.com')] 2024-09-21T04:00:57.359 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi089.front.sepia.ceph.com' 2024-09-21T04:00:57.360 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi089.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-21T04:00:57.432 DEBUG:teuthology.orchestra.run.smithi089:> true 2024-09-21T04:00:57.509 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi089.front.sepia.ceph.com' 2024-09-21T04:00:57.510 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi195.front.sepia.ceph.com' 2024-09-21T04:00:57.510 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi195.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-21T04:00:57.583 DEBUG:teuthology.orchestra.run.smithi195:> true 2024-09-21T04:00:57.619 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi195.front.sepia.ceph.com' 2024-09-21T04:00:57.619 INFO:teuthology.run_tasks:Running task clock... 2024-09-21T04:00:57.628 INFO:teuthology.task.clock:Syncing clocks and checking initial clock skew... 2024-09-21T04:00:57.629 INFO:teuthology.orchestra.run:Running command with timeout 360 2024-09-21T04:00:57.629 DEBUG:teuthology.orchestra.run.smithi089:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-09-21T04:00:57.631 INFO:teuthology.orchestra.run:Running command with timeout 360 2024-09-21T04:00:57.631 DEBUG:teuthology.orchestra.run.smithi195:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-09-21T04:00:57.658 INFO:teuthology.orchestra.run.smithi089.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2024-09-21T04:00:57.672 INFO:teuthology.orchestra.run.smithi089.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2024-09-21T04:00:57.689 INFO:teuthology.orchestra.run.smithi195.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2024-09-21T04:00:57.695 INFO:teuthology.orchestra.run.smithi089.stderr:sudo: ntpd: command not found 2024-09-21T04:00:57.703 INFO:teuthology.orchestra.run.smithi195.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2024-09-21T04:00:57.706 INFO:teuthology.orchestra.run.smithi089.stdout:506 Cannot talk to daemon 2024-09-21T04:00:57.720 INFO:teuthology.orchestra.run.smithi089.stderr:Failed to start ntp.service: Unit ntp.service not found. 2024-09-21T04:00:57.729 INFO:teuthology.orchestra.run.smithi195.stderr:sudo: ntpd: command not found 2024-09-21T04:00:57.734 INFO:teuthology.orchestra.run.smithi089.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2024-09-21T04:00:57.740 INFO:teuthology.orchestra.run.smithi195.stdout:506 Cannot talk to daemon 2024-09-21T04:00:57.754 INFO:teuthology.orchestra.run.smithi195.stderr:Failed to start ntp.service: Unit ntp.service not found. 2024-09-21T04:00:57.768 INFO:teuthology.orchestra.run.smithi195.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2024-09-21T04:00:57.787 INFO:teuthology.orchestra.run.smithi089.stderr:bash: line 1: ntpq: command not found 2024-09-21T04:00:57.790 INFO:teuthology.orchestra.run.smithi089.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-09-21T04:00:57.790 INFO:teuthology.orchestra.run.smithi089.stdout:=============================================================================== 2024-09-21T04:00:57.790 INFO:teuthology.orchestra.run.smithi089.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-21T04:00:57.790 INFO:teuthology.orchestra.run.smithi089.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-21T04:00:57.791 INFO:teuthology.orchestra.run.smithi089.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-21T04:00:57.791 INFO:teuthology.orchestra.run.smithi089.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-21T04:00:57.826 INFO:teuthology.orchestra.run.smithi195.stderr:bash: line 1: ntpq: command not found 2024-09-21T04:00:57.829 INFO:teuthology.orchestra.run.smithi195.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-09-21T04:00:57.830 INFO:teuthology.orchestra.run.smithi195.stdout:=============================================================================== 2024-09-21T04:00:57.830 INFO:teuthology.orchestra.run.smithi195.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-21T04:00:57.830 INFO:teuthology.orchestra.run.smithi195.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-21T04:00:57.830 INFO:teuthology.orchestra.run.smithi195.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-21T04:00:57.830 INFO:teuthology.orchestra.run.smithi195.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-21T04:00:57.831 INFO:teuthology.run_tasks:Running task cephadm... 2024-09-21T04:00:58.190 INFO:tasks.cephadm:Config: {'cephadm_branch': 'v17.2.0', 'cephadm_git_url': 'https://github.com/ceph/ceph', 'image': 'quay.io/ceph/ceph:v17.2.0', 'roleless': True, 'conf': {'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'debug ms': 1, 'debug osd': 20, 'osd shutdown pgref assert': True}}, 'flavor': 'default', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)', 'CEPHADM_REFRESH_FAILED'], 'log-only-match': ['CEPHADM_'], 'sha1': 'f36283e7c91cac1c01cb9a7c4ff21bd697dbd4f2'} 2024-09-21T04:00:58.191 INFO:tasks.cephadm:Cluster image is quay.io/ceph/ceph:v17.2.0 2024-09-21T04:00:58.191 INFO:tasks.cephadm:Cluster fsid is 1b9b79fc-77ce-11ef-baf5-efdc52797490 2024-09-21T04:00:58.192 INFO:tasks.cephadm:Choosing monitor IPs and ports... 2024-09-21T04:00:58.192 INFO:tasks.cephadm:No mon roles; fabricating mons 2024-09-21T04:00:58.192 INFO:tasks.cephadm:Monitor IPs: {'mon.smithi089': '172.21.15.89', 'mon.smithi195': '172.21.15.195'} 2024-09-21T04:00:58.192 INFO:tasks.cephadm:Normalizing hostnames... 2024-09-21T04:00:58.193 DEBUG:teuthology.orchestra.run.smithi089:> sudo hostname $(hostname -s) 2024-09-21T04:00:58.220 DEBUG:teuthology.orchestra.run.smithi195:> sudo hostname $(hostname -s) 2024-09-21T04:00:58.247 INFO:tasks.cephadm:Downloading cephadm (repo https://github.com/ceph/ceph ref v17.2.0)... 2024-09-21T04:00:58.247 DEBUG:teuthology.orchestra.run.smithi089:> curl --silent https://raw.githubusercontent.com/ceph/ceph/v17.2.0/src/cephadm/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2024-09-21T04:00:58.514 INFO:teuthology.orchestra.run.smithi089.stdout:-rw-r--r--. 1 ubuntu ubuntu 320521 Sep 21 04:00 /home/ubuntu/cephtest/cephadm 2024-09-21T04:00:58.514 DEBUG:teuthology.orchestra.run.smithi195:> curl --silent https://raw.githubusercontent.com/ceph/ceph/v17.2.0/src/cephadm/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2024-09-21T04:00:58.635 INFO:teuthology.orchestra.run.smithi195.stdout:-rw-r--r--. 1 ubuntu ubuntu 320521 Sep 21 04:00 /home/ubuntu/cephtest/cephadm 2024-09-21T04:00:58.636 DEBUG:teuthology.orchestra.run.smithi089:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2024-09-21T04:00:58.652 DEBUG:teuthology.orchestra.run.smithi195:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2024-09-21T04:00:58.682 INFO:tasks.cephadm:Pulling image quay.io/ceph/ceph:v17.2.0 on all hosts... 2024-09-21T04:00:58.682 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 pull 2024-09-21T04:00:58.695 DEBUG:teuthology.orchestra.run.smithi195:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 pull 2024-09-21T04:01:00.317 INFO:teuthology.orchestra.run.smithi089.stderr:Pulling container image quay.io/ceph/ceph:v17.2.0... 2024-09-21T04:01:00.375 INFO:teuthology.orchestra.run.smithi195.stderr:Pulling container image quay.io/ceph/ceph:v17.2.0... 2024-09-21T04:01:11.979 INFO:teuthology.orchestra.run.smithi195.stdout:{ 2024-09-21T04:01:11.979 INFO:teuthology.orchestra.run.smithi195.stdout: "ceph_version": "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)", 2024-09-21T04:01:11.979 INFO:teuthology.orchestra.run.smithi195.stdout: "image_id": "e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9", 2024-09-21T04:01:11.980 INFO:teuthology.orchestra.run.smithi195.stdout: "repo_digests": [ 2024-09-21T04:01:11.980 INFO:teuthology.orchestra.run.smithi195.stdout: "quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a", 2024-09-21T04:01:11.980 INFO:teuthology.orchestra.run.smithi195.stdout: "quay.io/ceph/ceph@sha256:cb4d698cb769b6aba05bf6ef04f41a7fe694160140347576e13bd9348514b667" 2024-09-21T04:01:11.980 INFO:teuthology.orchestra.run.smithi195.stdout: ] 2024-09-21T04:01:11.980 INFO:teuthology.orchestra.run.smithi195.stdout:} 2024-09-21T04:01:12.177 INFO:teuthology.orchestra.run.smithi089.stdout:{ 2024-09-21T04:01:12.177 INFO:teuthology.orchestra.run.smithi089.stdout: "ceph_version": "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)", 2024-09-21T04:01:12.177 INFO:teuthology.orchestra.run.smithi089.stdout: "image_id": "e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9", 2024-09-21T04:01:12.177 INFO:teuthology.orchestra.run.smithi089.stdout: "repo_digests": [ 2024-09-21T04:01:12.177 INFO:teuthology.orchestra.run.smithi089.stdout: "quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a", 2024-09-21T04:01:12.177 INFO:teuthology.orchestra.run.smithi089.stdout: "quay.io/ceph/ceph@sha256:cb4d698cb769b6aba05bf6ef04f41a7fe694160140347576e13bd9348514b667" 2024-09-21T04:01:12.177 INFO:teuthology.orchestra.run.smithi089.stdout: ] 2024-09-21T04:01:12.178 INFO:teuthology.orchestra.run.smithi089.stdout:} 2024-09-21T04:01:12.195 DEBUG:teuthology.orchestra.run.smithi089:> sudo mkdir -p /etc/ceph 2024-09-21T04:01:12.261 DEBUG:teuthology.orchestra.run.smithi195:> sudo mkdir -p /etc/ceph 2024-09-21T04:01:12.291 DEBUG:teuthology.orchestra.run.smithi089:> sudo chmod 777 /etc/ceph 2024-09-21T04:01:12.316 DEBUG:teuthology.orchestra.run.smithi195:> sudo chmod 777 /etc/ceph 2024-09-21T04:01:12.358 INFO:tasks.cephadm:Writing seed config... 2024-09-21T04:01:12.372 INFO:tasks.cephadm: override: [mgr] debug mgr = 20 2024-09-21T04:01:12.372 INFO:tasks.cephadm: override: [mgr] debug ms = 1 2024-09-21T04:01:12.372 INFO:tasks.cephadm: override: [mon] debug mon = 20 2024-09-21T04:01:12.372 INFO:tasks.cephadm: override: [mon] debug ms = 1 2024-09-21T04:01:12.372 INFO:tasks.cephadm: override: [mon] debug paxos = 20 2024-09-21T04:01:12.372 INFO:tasks.cephadm: override: [osd] debug ms = 1 2024-09-21T04:01:12.372 INFO:tasks.cephadm: override: [osd] debug osd = 20 2024-09-21T04:01:12.372 INFO:tasks.cephadm: override: [osd] osd shutdown pgref assert = True 2024-09-21T04:01:12.373 DEBUG:teuthology.orchestra.run.smithi089:> set -ex 2024-09-21T04:01:12.373 DEBUG:teuthology.orchestra.run.smithi089:> dd of=/home/ubuntu/cephtest/seed.ceph.conf 2024-09-21T04:01:12.388 DEBUG:tasks.cephadm:Final config: [global] # make logging friendly to teuthology log_to_file = true log_to_stderr = false log to journald = false mon cluster log to file = true mon cluster log file level = debug mon clock drift allowed = 1.000 # replicate across OSDs, not hosts osd crush chooseleaf type = 0 #osd pool default size = 2 osd pool default erasure code profile = plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd # enable some debugging auth debug = true ms die on old message = true ms die on bug = true debug asserts on shutdown = true # adjust warnings mon max pg per osd = 10000# >= luminous mon pg warn max object skew = 0 mon osd allow primary affinity = true mon osd allow pg remap = true mon warn on legacy crush tunables = false mon warn on crush straw calc version zero = false mon warn on no sortbitwise = false mon warn on osd down out interval zero = false mon warn on too few osds = false mon_warn_on_pool_pg_num_not_power_of_two = false # disable pg_autoscaler by default for new pools osd_pool_default_pg_autoscale_mode = off # tests delete pools mon allow pool delete = true fsid = 1b9b79fc-77ce-11ef-baf5-efdc52797490 [osd] osd scrub load threshold = 5.0 osd scrub max interval = 600 osd mclock profile = high_recovery_ops osd recover clone overlap = true osd recovery max chunk = 1048576 osd deep scrub update digest min age = 30 osd map max advance = 10 osd memory target autotune = true # debugging osd debug shutdown = true osd debug op order = true osd debug verify stray on activate = true osd debug pg log writeout = true osd debug verify cached snaps = true osd debug verify missing on start = true osd debug misdirected ops = true osd op queue = debug_random osd op queue cut off = debug_random osd shutdown pgref assert = True bdev debug aio = true osd sloppy crc = true debug ms = 1 debug osd = 20 [mgr] mon reweight min pgs per osd = 4 mon reweight min bytes per osd = 10 mgr/telemetry/nag = false debug mgr = 20 debug ms = 1 [mon] mon data avail warn = 5 mon mgr mkfs grace = 240 mon reweight min pgs per osd = 4 mon osd reporter subtree level = osd mon osd prime pg temp = true mon reweight min bytes per osd = 10 # rotate auth tickets quickly to exercise renewal paths auth mon ticket ttl = 660# 11m auth service ticket ttl = 240# 4m # don't complain about global id reclaim mon_warn_on_insecure_global_id_reclaim = false mon_warn_on_insecure_global_id_reclaim_allowed = false debug mon = 20 debug ms = 1 debug paxos = 20 [client.rgw] rgw cache enabled = true rgw enable ops log = true rgw enable usage log = true 2024-09-21T04:01:12.388 DEBUG:teuthology.orchestra.run.smithi089:mon.smithi089> sudo journalctl -f -n 0 -u ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490@mon.smithi089.service 2024-09-21T04:01:12.430 INFO:tasks.cephadm:Bootstrapping... 2024-09-21T04:01:12.430 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 -v bootstrap --fsid 1b9b79fc-77ce-11ef-baf5-efdc52797490 --config /home/ubuntu/cephtest/seed.ceph.conf --output-config /etc/ceph/ceph.conf --output-keyring /etc/ceph/ceph.client.admin.keyring --output-pub-ssh-key /home/ubuntu/cephtest/ceph.pub --mon-ip 172.21.15.89 --skip-admin-label && sudo chmod +r /etc/ceph/ceph.client.admin.keyring 2024-09-21T04:01:12.617 INFO:teuthology.orchestra.run.smithi089.stderr:-------------------------------------------------------------------------------- 2024-09-21T04:01:12.618 INFO:teuthology.orchestra.run.smithi089.stderr:cephadm ['--image', 'quay.io/ceph/ceph:v17.2.0', '-v', 'bootstrap', '--fsid', '1b9b79fc-77ce-11ef-baf5-efdc52797490', '--config', '/home/ubuntu/cephtest/seed.ceph.conf', '--output-config', '/etc/ceph/ceph.conf', '--output-keyring', '/etc/ceph/ceph.client.admin.keyring', '--output-pub-ssh-key', '/home/ubuntu/cephtest/ceph.pub', '--mon-ip', '172.21.15.89', '--skip-admin-label'] 2024-09-21T04:01:12.639 INFO:teuthology.orchestra.run.smithi089.stderr:/bin/podman: 5.2.2 2024-09-21T04:01:12.641 INFO:teuthology.orchestra.run.smithi089.stderr:Verifying podman|docker is present... 2024-09-21T04:01:12.661 INFO:teuthology.orchestra.run.smithi089.stderr:/bin/podman: 5.2.2 2024-09-21T04:01:12.663 INFO:teuthology.orchestra.run.smithi089.stderr:Verifying lvm2 is present... 2024-09-21T04:01:12.663 INFO:teuthology.orchestra.run.smithi089.stderr:Verifying time synchronization is in place... 2024-09-21T04:01:12.670 INFO:teuthology.orchestra.run.smithi089.stderr:systemctl: Failed to get unit file state for chrony.service: No such file or directory 2024-09-21T04:01:12.676 INFO:teuthology.orchestra.run.smithi089.stderr:systemctl: inactive 2024-09-21T04:01:12.683 INFO:teuthology.orchestra.run.smithi089.stderr:systemctl: enabled 2024-09-21T04:01:12.689 INFO:teuthology.orchestra.run.smithi089.stderr:systemctl: active 2024-09-21T04:01:12.690 INFO:teuthology.orchestra.run.smithi089.stderr:Unit chronyd.service is enabled and running 2024-09-21T04:01:12.690 INFO:teuthology.orchestra.run.smithi089.stderr:Repeating the final host check... 2024-09-21T04:01:12.711 INFO:teuthology.orchestra.run.smithi089.stderr:/bin/podman: 5.2.2 2024-09-21T04:01:12.713 INFO:teuthology.orchestra.run.smithi089.stderr:podman (/bin/podman) version 5.2.2 is present 2024-09-21T04:01:12.713 INFO:teuthology.orchestra.run.smithi089.stderr:systemctl is present 2024-09-21T04:01:12.713 INFO:teuthology.orchestra.run.smithi089.stderr:lvcreate is present 2024-09-21T04:01:12.719 INFO:teuthology.orchestra.run.smithi089.stderr:systemctl: Failed to get unit file state for chrony.service: No such file or directory 2024-09-21T04:01:12.725 INFO:teuthology.orchestra.run.smithi089.stderr:systemctl: inactive 2024-09-21T04:01:12.732 INFO:teuthology.orchestra.run.smithi089.stderr:systemctl: enabled 2024-09-21T04:01:12.738 INFO:teuthology.orchestra.run.smithi089.stderr:systemctl: active 2024-09-21T04:01:12.739 INFO:teuthology.orchestra.run.smithi089.stderr:Unit chronyd.service is enabled and running 2024-09-21T04:01:12.739 INFO:teuthology.orchestra.run.smithi089.stderr:Host looks OK 2024-09-21T04:01:12.739 INFO:teuthology.orchestra.run.smithi089.stderr:Cluster fsid: 1b9b79fc-77ce-11ef-baf5-efdc52797490 2024-09-21T04:01:12.739 INFO:teuthology.orchestra.run.smithi089.stderr:Acquiring lock 140034244175760 on /run/cephadm/1b9b79fc-77ce-11ef-baf5-efdc52797490.lock 2024-09-21T04:01:12.740 INFO:teuthology.orchestra.run.smithi089.stderr:Lock 140034244175760 acquired on /run/cephadm/1b9b79fc-77ce-11ef-baf5-efdc52797490.lock 2024-09-21T04:01:12.740 INFO:teuthology.orchestra.run.smithi089.stderr:Verifying IP 172.21.15.89 port 3300 ... 2024-09-21T04:01:12.740 INFO:teuthology.orchestra.run.smithi089.stderr:Verifying IP 172.21.15.89 port 6789 ... 2024-09-21T04:01:12.740 INFO:teuthology.orchestra.run.smithi089.stderr:Base mon IP is 172.21.15.89, final addrv is [v2:172.21.15.89:3300,v1:172.21.15.89:6789] 2024-09-21T04:01:12.743 INFO:teuthology.orchestra.run.smithi089.stderr:/sbin/ip: default via 172.21.15.254 dev enp3s0f1 proto dhcp src 172.21.15.89 metric 100 2024-09-21T04:01:12.743 INFO:teuthology.orchestra.run.smithi089.stderr:/sbin/ip: 172.21.0.0/20 dev enp3s0f1 proto kernel scope link src 172.21.15.89 metric 100 2024-09-21T04:01:12.745 INFO:teuthology.orchestra.run.smithi089.stderr:/sbin/ip: ::1 dev lo proto kernel metric 256 pref medium 2024-09-21T04:01:12.746 INFO:teuthology.orchestra.run.smithi089.stderr:/sbin/ip: fe80::/64 dev enp3s0f1 proto kernel metric 1024 pref medium 2024-09-21T04:01:12.746 INFO:teuthology.orchestra.run.smithi089.stderr:/sbin/ip: default via fe80::327c:5e00:6487:71e0 dev enp3s0f1 proto ra metric 100 pref medium 2024-09-21T04:01:12.748 INFO:teuthology.orchestra.run.smithi089.stderr:/sbin/ip: 1: lo: mtu 65536 state UNKNOWN qlen 1000 2024-09-21T04:01:12.748 INFO:teuthology.orchestra.run.smithi089.stderr:/sbin/ip: inet6 ::1/128 scope host 2024-09-21T04:01:12.748 INFO:teuthology.orchestra.run.smithi089.stderr:/sbin/ip: valid_lft forever preferred_lft forever 2024-09-21T04:01:12.748 INFO:teuthology.orchestra.run.smithi089.stderr:/sbin/ip: 5: enp3s0f1: mtu 1500 state UP qlen 1000 2024-09-21T04:01:12.749 INFO:teuthology.orchestra.run.smithi089.stderr:/sbin/ip: inet6 fe80::ec4:7aff:fe8f:d501/64 scope link noprefixroute 2024-09-21T04:01:12.749 INFO:teuthology.orchestra.run.smithi089.stderr:/sbin/ip: valid_lft forever preferred_lft forever 2024-09-21T04:01:12.749 INFO:teuthology.orchestra.run.smithi089.stderr:Mon IP `172.21.15.89` is in CIDR network `172.21.0.0/20` 2024-09-21T04:01:12.749 INFO:teuthology.orchestra.run.smithi089.stderr:- internal network (--cluster-network) has not been provided, OSD replication will default to the public_network 2024-09-21T04:01:12.750 INFO:teuthology.orchestra.run.smithi089.stderr:Pulling container image quay.io/ceph/ceph:v17.2.0... 2024-09-21T04:01:12.848 INFO:teuthology.orchestra.run.smithi089.stderr:/bin/podman: Trying to pull quay.io/ceph/ceph:v17.2.0... 2024-09-21T04:01:13.219 INFO:teuthology.orchestra.run.smithi089.stderr:/bin/podman: Getting image source signatures 2024-09-21T04:01:13.219 INFO:teuthology.orchestra.run.smithi089.stderr:/bin/podman: Copying blob sha256:c32ab78b488d0c72f64eded765c0cf6b5bf2c75dab66cb62a9d367fa6ec42513 2024-09-21T04:01:13.219 INFO:teuthology.orchestra.run.smithi089.stderr:/bin/podman: Copying blob sha256:33ca8fff7868c4dc0c11e09bca97c720eb9cfbab7221216754367dd8de70388a 2024-09-21T04:01:13.219 INFO:teuthology.orchestra.run.smithi089.stderr:/bin/podman: Copying blob sha256:89b4a75bc2d8500f15463747507c9623df43886c134463e7f0527e70900e7a7b 2024-09-21T04:01:13.219 INFO:teuthology.orchestra.run.smithi089.stderr:/bin/podman: Copying blob sha256:a70843738bb77e1ab9c1f85969ebdfa55f178e746be081d1cb4f94011f69eb7c 2024-09-21T04:01:13.219 INFO:teuthology.orchestra.run.smithi089.stderr:/bin/podman: Copying blob sha256:599d07cb321ff0a3c82224e1138fc685793fa69b93ed5780415751a5f7e4b8c2 2024-09-21T04:01:13.219 INFO:teuthology.orchestra.run.smithi089.stderr:/bin/podman: Copying config sha256:e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 2024-09-21T04:01:13.372 INFO:teuthology.orchestra.run.smithi089.stderr:/bin/podman: Writing manifest to image destination 2024-09-21T04:01:13.525 INFO:teuthology.orchestra.run.smithi089.stderr:/bin/podman: e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 2024-09-21T04:01:13.950 INFO:teuthology.orchestra.run.smithi089.stderr:ceph: ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable) 2024-09-21T04:01:14.492 INFO:teuthology.orchestra.run.smithi089.stderr:Ceph version: ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable) 2024-09-21T04:01:14.492 INFO:teuthology.orchestra.run.smithi089.stderr:Extracting ceph user uid/gid from container image... 2024-09-21T04:01:14.908 INFO:teuthology.orchestra.run.smithi089.stderr:stat: 167 167 2024-09-21T04:01:15.440 INFO:teuthology.orchestra.run.smithi089.stderr:Creating initial keys... 2024-09-21T04:01:18.199 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-authtool: AQANRe5mpD/QGxAAjiditYCRtpN5Z4A3UJzeOg== 2024-09-21T04:01:25.922 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-authtool: AQAVRe5mugEPMxAAF1XOCAkUNy5hCfKEN34GcQ== 2024-09-21T04:01:27.062 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-authtool: AQAWRe5mgw+3ORAANL6ddNxkfkdo3NILhjo00Q== 2024-09-21T04:01:27.710 INFO:teuthology.orchestra.run.smithi089.stderr:Creating initial monmap... 2024-09-21T04:01:28.102 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/monmaptool: /usr/bin/monmaptool: monmap file /tmp/monmap 2024-09-21T04:01:28.102 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/monmaptool: setting min_mon_release = octopus 2024-09-21T04:01:28.102 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/monmaptool: /usr/bin/monmaptool: set fsid to 1b9b79fc-77ce-11ef-baf5-efdc52797490 2024-09-21T04:01:28.102 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/monmaptool: /usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2024-09-21T04:01:28.608 INFO:teuthology.orchestra.run.smithi089.stderr:monmaptool for smithi089 [v2:172.21.15.89:3300,v1:172.21.15.89:6789] on /usr/bin/monmaptool: monmap file /tmp/monmap 2024-09-21T04:01:28.608 INFO:teuthology.orchestra.run.smithi089.stderr:setting min_mon_release = octopus 2024-09-21T04:01:28.608 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/monmaptool: set fsid to 1b9b79fc-77ce-11ef-baf5-efdc52797490 2024-09-21T04:01:28.608 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2024-09-21T04:01:28.608 INFO:teuthology.orchestra.run.smithi089.stderr: 2024-09-21T04:01:28.608 INFO:teuthology.orchestra.run.smithi089.stderr:Creating mon... 2024-09-21T04:01:30.116 INFO:teuthology.orchestra.run.smithi089.stderr:create mon.smithi089 on 2024-09-21T04:01:30.330 INFO:teuthology.orchestra.run.smithi089.stderr:systemctl: Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target → /etc/systemd/system/ceph.target. 2024-09-21T04:01:30.484 INFO:teuthology.orchestra.run.smithi089.stderr:systemctl: Created symlink /etc/systemd/system/multi-user.target.wants/ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490.target → /etc/systemd/system/ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490.target. 2024-09-21T04:01:30.484 INFO:teuthology.orchestra.run.smithi089.stderr:systemctl: Created symlink /etc/systemd/system/ceph.target.wants/ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490.target → /etc/systemd/system/ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490.target. 2024-09-21T04:01:30.776 INFO:teuthology.orchestra.run.smithi089.stderr:systemctl: Failed to reset failed state of unit ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490@mon.smithi089.service: Unit ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490@mon.smithi089.service not loaded. 2024-09-21T04:01:30.783 INFO:teuthology.orchestra.run.smithi089.stderr:systemctl: Created symlink /etc/systemd/system/ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490.target.wants/ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490@mon.smithi089.service → /etc/systemd/system/ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490@.service. 2024-09-21T04:01:31.515 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:31 smithi089 podman[23377]: 2024-09-21 04:01:31.077580966 +0000 UTC m=+0.016592452 image pull e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 quay.io/ceph/ceph:v17.2.0 2024-09-21T04:01:32.264 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:31 smithi089 podman[23377]: 2024-09-21 04:01:31.99623195 +0000 UTC m=+0.935243469 container create da0ced9603d29371c33e7d318c38fbe50275c13a89fecf6f8f3a8aee67b5b282 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-mon-smithi089, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_BRANCH=HEAD, architecture=x86_64, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.openshift.expose-services=, io.k8s.display-name=CentOS Stream 8, io.buildah.version=1.19.8, ceph=True, release=754, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_REPO=https://github.com/ceph/ceph-container.git, maintainer=Guillaume Abrioux , io.openshift.tags=base centos centos-stream, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, distribution-scope=public, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vcs-type=git, version=8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.component=centos-stream-container, vendor=Red Hat, Inc., GIT_CLEAN=True, name=centos-stream, CEPH_POINT_RELEASE=-17.2.0, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, RELEASE=HEAD, build-date=2022-05-03T08:36:31.336870, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly.) 2024-09-21T04:01:32.265 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:32 smithi089 podman[23377]: 2024-09-21 04:01:32.215325199 +0000 UTC m=+1.154336680 container init da0ced9603d29371c33e7d318c38fbe50275c13a89fecf6f8f3a8aee67b5b282 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-mon-smithi089, GIT_BRANCH=HEAD, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, maintainer=Guillaume Abrioux , GIT_CLEAN=True, io.k8s.display-name=CentOS Stream 8, architecture=x86_64, io.openshift.expose-services=, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, RELEASE=HEAD, vcs-type=git, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, ceph=True, release=754, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.tags=base centos centos-stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, build-date=2022-05-03T08:36:31.336870, CEPH_POINT_RELEASE=-17.2.0, distribution-scope=public, version=8, io.buildah.version=1.19.8, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.component=centos-stream-container, vendor=Red Hat, Inc., name=centos-stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com) 2024-09-21T04:01:32.265 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:32 smithi089 podman[23377]: 2024-09-21 04:01:32.2189433 +0000 UTC m=+1.157954774 container start da0ced9603d29371c33e7d318c38fbe50275c13a89fecf6f8f3a8aee67b5b282 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-mon-smithi089, build-date=2022-05-03T08:36:31.336870, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.k8s.display-name=CentOS Stream 8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, CEPH_POINT_RELEASE=-17.2.0, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vendor=Red Hat, Inc., RELEASE=HEAD, GIT_BRANCH=HEAD, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vcs-type=git, io.buildah.version=1.19.8, GIT_CLEAN=True, name=centos-stream, release=754, maintainer=Guillaume Abrioux , io.openshift.expose-services=, version=8, ceph=True, distribution-scope=public, architecture=x86_64, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, com.redhat.component=centos-stream-container, io.openshift.tags=base centos centos-stream, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754) 2024-09-21T04:01:32.300 INFO:teuthology.orchestra.run.smithi089.stderr:systemctl: disabled 2024-09-21T04:01:32.307 INFO:teuthology.orchestra.run.smithi089.stderr:systemctl: inactive 2024-09-21T04:01:32.308 INFO:teuthology.orchestra.run.smithi089.stderr:firewalld.service is not enabled 2024-09-21T04:01:32.308 INFO:teuthology.orchestra.run.smithi089.stderr:Not possible to enable service . firewalld.service is not available 2024-09-21T04:01:32.308 INFO:teuthology.orchestra.run.smithi089.stderr:Waiting for mon to start... 2024-09-21T04:01:32.309 INFO:teuthology.orchestra.run.smithi089.stderr:Waiting for mon... 2024-09-21T04:01:32.534 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:32 smithi089 ceph-mon[23396]: mkfs 1b9b79fc-77ce-11ef-baf5-efdc52797490 2024-09-21T04:01:32.534 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:32 smithi089 ceph-mon[23396]: mon.smithi089 is new leader, mons smithi089 in quorum (ranks 0) 2024-09-21T04:01:32.534 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:32 smithi089 bash[23377]: da0ced9603d29371c33e7d318c38fbe50275c13a89fecf6f8f3a8aee67b5b282 2024-09-21T04:01:32.534 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:32 smithi089 systemd[1]: Started Ceph mon.smithi089 for 1b9b79fc-77ce-11ef-baf5-efdc52797490. 2024-09-21T04:01:32.858 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: cluster: 2024-09-21T04:01:32.858 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: id: 1b9b79fc-77ce-11ef-baf5-efdc52797490 2024-09-21T04:01:32.858 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: health: HEALTH_OK 2024-09-21T04:01:32.858 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: 2024-09-21T04:01:32.858 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: services: 2024-09-21T04:01:32.858 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: mon: 1 daemons, quorum smithi089 (age 0.596825s) 2024-09-21T04:01:32.858 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: mgr: no daemons active 2024-09-21T04:01:32.858 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: osd: 0 osds: 0 up, 0 in 2024-09-21T04:01:32.858 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: 2024-09-21T04:01:32.859 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: data: 2024-09-21T04:01:32.859 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: pools: 0 pools, 0 pgs 2024-09-21T04:01:32.859 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: objects: 0 objects, 0 B 2024-09-21T04:01:32.859 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: usage: 0 B used, 0 B / 0 B avail 2024-09-21T04:01:32.859 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: pgs: 2024-09-21T04:01:32.859 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: 2024-09-21T04:01:33.402 INFO:teuthology.orchestra.run.smithi089.stderr:mon is available 2024-09-21T04:01:33.402 INFO:teuthology.orchestra.run.smithi089.stderr:Assimilating anything we can from ceph.conf... 2024-09-21T04:01:33.514 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:33 smithi089 ceph-mon[23396]: mon.smithi089 is new leader, mons smithi089 in quorum (ranks 0) 2024-09-21T04:01:33.514 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:33 smithi089 ceph-mon[23396]: monmap e1: 1 mons at {smithi089=[v2:172.21.15.89:3300/0,v1:172.21.15.89:6789/0]} 2024-09-21T04:01:33.514 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:33 smithi089 ceph-mon[23396]: fsmap 2024-09-21T04:01:33.514 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:33 smithi089 ceph-mon[23396]: osdmap e1: 0 total, 0 up, 0 in 2024-09-21T04:01:33.514 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:33 smithi089 ceph-mon[23396]: mgrmap e1: no daemons active 2024-09-21T04:01:33.515 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:33 smithi089 ceph-mon[23396]: from='client.? 172.21.15.89:0/2625305603' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-09-21T04:01:33.906 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: 2024-09-21T04:01:33.906 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: [global] 2024-09-21T04:01:33.906 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: fsid = 1b9b79fc-77ce-11ef-baf5-efdc52797490 2024-09-21T04:01:33.906 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: mon_host = [v2:172.21.15.89:3300,v1:172.21.15.89:6789] 2024-09-21T04:01:33.907 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: mon_osd_allow_pg_remap = true 2024-09-21T04:01:33.907 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: mon_osd_allow_primary_affinity = true 2024-09-21T04:01:33.907 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: mon_warn_on_no_sortbitwise = false 2024-09-21T04:01:33.907 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: osd_crush_chooseleaf_type = 0 2024-09-21T04:01:33.907 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: 2024-09-21T04:01:33.907 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: [mgr] 2024-09-21T04:01:33.907 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: mgr/telemetry/nag = false 2024-09-21T04:01:33.907 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: 2024-09-21T04:01:33.907 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: [osd] 2024-09-21T04:01:33.907 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: osd_map_max_advance = 10 2024-09-21T04:01:33.907 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: osd_sloppy_crc = true 2024-09-21T04:01:34.514 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:34 smithi089 ceph-mon[23396]: from='client.? 172.21.15.89:0/3469220808' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2024-09-21T04:01:34.514 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:34 smithi089 ceph-mon[23396]: from='client.? 172.21.15.89:0/3469220808' entity='client.admin' cmd='[{"prefix": "config assimilate-conf"}]': finished 2024-09-21T04:01:34.516 INFO:teuthology.orchestra.run.smithi089.stderr:Generating new minimal ceph.conf... 2024-09-21T04:01:35.546 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:35 smithi089 ceph-mon[23396]: from='client.? 172.21.15.89:0/3031493912' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T04:01:35.549 INFO:teuthology.orchestra.run.smithi089.stderr:Restarting the monitor... 2024-09-21T04:01:35.952 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:35 smithi089 systemd[1]: Stopping Ceph mon.smithi089 for 1b9b79fc-77ce-11ef-baf5-efdc52797490... 2024-09-21T04:01:35.953 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:35 smithi089 bash[23613]: Error: no container with name or ID "ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-mon.smithi089" found: no such container 2024-09-21T04:01:35.953 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:35 smithi089 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-mon-smithi089[23391]: 2024-09-21T04:01:35.714+0000 7f91a312b700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.smithi089 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-09-21T04:01:35.953 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:35 smithi089 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-mon-smithi089[23391]: 2024-09-21T04:01:35.714+0000 7f91a312b700 -1 mon.smithi089@0(leader) e1 *** Got Signal Terminated *** 2024-09-21T04:01:36.225 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:35 smithi089 podman[23624]: 2024-09-21 04:01:35.950542144 +0000 UTC m=+0.348422764 container died da0ced9603d29371c33e7d318c38fbe50275c13a89fecf6f8f3a8aee67b5b282 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-mon-smithi089, CEPH_POINT_RELEASE=-17.2.0, vcs-type=git, io.k8s.display-name=CentOS Stream 8, build-date=2022-05-03T08:36:31.336870, io.buildah.version=1.19.8, version=8, maintainer=Guillaume Abrioux , com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_BRANCH=HEAD, GIT_CLEAN=True, architecture=x86_64, com.redhat.component=centos-stream-container, ceph=True, distribution-scope=public, io.openshift.expose-services=, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_REPO=https://github.com/ceph/ceph-container.git, vendor=Red Hat, Inc., io.openshift.tags=base centos centos-stream, RELEASE=HEAD, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, release=754, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, name=centos-stream) 2024-09-21T04:01:36.514 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:36 smithi089 podman[23624]: 2024-09-21 04:01:36.305938067 +0000 UTC m=+0.703818719 container cleanup da0ced9603d29371c33e7d318c38fbe50275c13a89fecf6f8f3a8aee67b5b282 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-mon-smithi089, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, CEPH_POINT_RELEASE=-17.2.0, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_BRANCH=HEAD, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., distribution-scope=public, name=centos-stream, GIT_REPO=https://github.com/ceph/ceph-container.git, build-date=2022-05-03T08:36:31.336870, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, ceph=True, io.openshift.expose-services=, GIT_CLEAN=True, version=8, maintainer=Guillaume Abrioux , vendor=Red Hat, Inc., description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.component=centos-stream-container, io.openshift.tags=base centos centos-stream, architecture=x86_64, release=754, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, RELEASE=HEAD, vcs-type=git, io.k8s.display-name=CentOS Stream 8, io.buildah.version=1.19.8) 2024-09-21T04:01:36.514 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:36 smithi089 bash[23624]: ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-mon-smithi089 2024-09-21T04:01:36.515 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:36 smithi089 bash[23650]: Error: no container with name or ID "ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-mon.smithi089" found: no such container 2024-09-21T04:01:36.913 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:36 smithi089 podman[23636]: 2024-09-21 04:01:36.590350665 +0000 UTC m=+0.640114347 container remove da0ced9603d29371c33e7d318c38fbe50275c13a89fecf6f8f3a8aee67b5b282 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-mon-smithi089, version=8, architecture=x86_64, CEPH_POINT_RELEASE=-17.2.0, build-date=2022-05-03T08:36:31.336870, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.openshift.expose-services=, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_BRANCH=HEAD, distribution-scope=public, name=centos-stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vendor=Red Hat, Inc., com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_REPO=https://github.com/ceph/ceph-container.git, maintainer=Guillaume Abrioux , com.redhat.component=centos-stream-container, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-type=git, io.k8s.display-name=CentOS Stream 8, io.openshift.tags=base centos centos-stream, RELEASE=HEAD, ceph=True, release=754, io.buildah.version=1.19.8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_CLEAN=True) 2024-09-21T04:01:36.913 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:36 smithi089 systemd[1]: ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490@mon.smithi089.service: Deactivated successfully. 2024-09-21T04:01:36.914 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:36 smithi089 systemd[1]: Stopped Ceph mon.smithi089 for 1b9b79fc-77ce-11ef-baf5-efdc52797490. 2024-09-21T04:01:36.914 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:36 smithi089 systemd[1]: Starting Ceph mon.smithi089 for 1b9b79fc-77ce-11ef-baf5-efdc52797490... 2024-09-21T04:01:36.914 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:36 smithi089 podman[23715]: 2024-09-21 04:01:36.721133267 +0000 UTC m=+0.016318668 image pull e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 quay.io/ceph/ceph:v17.2.0 2024-09-21T04:01:36.914 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:36 smithi089 podman[23715]: 2024-09-21 04:01:36.824760786 +0000 UTC m=+0.119946239 container create 1d2b3719e46efeb28714f3f5bdfa041021ff4c3c0f21316359a357c8b7c7b7d9 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-mon-smithi089, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_BRANCH=HEAD, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, CEPH_POINT_RELEASE=-17.2.0, vcs-type=git, ceph=True, com.redhat.component=centos-stream-container, RELEASE=HEAD, maintainer=Guillaume Abrioux , description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vendor=Red Hat, Inc., version=8, io.openshift.tags=base centos centos-stream, GIT_CLEAN=True, io.openshift.expose-services=, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_REPO=https://github.com/ceph/ceph-container.git, io.k8s.display-name=CentOS Stream 8, io.buildah.version=1.19.8, distribution-scope=public, name=centos-stream, build-date=2022-05-03T08:36:31.336870, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, release=754, architecture=x86_64) 2024-09-21T04:01:37.131 INFO:teuthology.orchestra.run.smithi089.stderr:Setting mon public_network to 172.21.0.0/20 2024-09-21T04:01:37.267 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:37 smithi089 podman[23715]: 2024-09-21 04:01:37.0104819 +0000 UTC m=+0.305667301 container init 1d2b3719e46efeb28714f3f5bdfa041021ff4c3c0f21316359a357c8b7c7b7d9 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-mon-smithi089, io.openshift.tags=base centos centos-stream, release=754, GIT_CLEAN=True, com.redhat.component=centos-stream-container, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, architecture=x86_64, build-date=2022-05-03T08:36:31.336870, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vendor=Red Hat, Inc., maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, io.buildah.version=1.19.8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., ceph=True, CEPH_POINT_RELEASE=-17.2.0, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.k8s.display-name=CentOS Stream 8, io.openshift.expose-services=, distribution-scope=public, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, name=centos-stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, RELEASE=HEAD, vcs-type=git, GIT_REPO=https://github.com/ceph/ceph-container.git, version=8) 2024-09-21T04:01:37.267 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:37 smithi089 podman[23715]: 2024-09-21 04:01:37.013822551 +0000 UTC m=+0.309007945 container start 1d2b3719e46efeb28714f3f5bdfa041021ff4c3c0f21316359a357c8b7c7b7d9 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-mon-smithi089, io.openshift.expose-services=, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.openshift.tags=base centos centos-stream, GIT_BRANCH=HEAD, ceph=True, maintainer=Guillaume Abrioux , version=8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.buildah.version=1.19.8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vcs-type=git, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.component=centos-stream-container, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, vendor=Red Hat, Inc., GIT_CLEAN=True, name=centos-stream, GIT_REPO=https://github.com/ceph/ceph-container.git, release=754, architecture=x86_64, RELEASE=HEAD, distribution-scope=public, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, CEPH_POINT_RELEASE=-17.2.0, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.k8s.display-name=CentOS Stream 8, build-date=2022-05-03T08:36:31.336870) 2024-09-21T04:01:37.267 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:37 smithi089 ceph-mon[23733]: set uid:gid to 167:167 (ceph:ceph) 2024-09-21T04:01:37.267 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:37 smithi089 ceph-mon[23733]: ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable), process ceph-mon, pid 2 2024-09-21T04:01:37.267 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:37 smithi089 ceph-mon[23733]: pidfile_write: ignore empty --pid-file 2024-09-21T04:01:37.267 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:37 smithi089 ceph-mon[23733]: load: jerasure load: lrc 2024-09-21T04:01:37.267 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:37 smithi089 ceph-mon[23733]: rocksdb: RocksDB version: 6.15.5 2024-09-21T04:01:37.267 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:37 smithi089 ceph-mon[23733]: rocksdb: Git sha rocksdb_build_git_sha:@0@ 2024-09-21T04:01:37.268 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:37 smithi089 ceph-mon[23733]: rocksdb: Compile date Apr 18 2022 2024-09-21T04:01:37.268 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:37 smithi089 ceph-mon[23733]: rocksdb: DB SUMMARY 2024-09-21T04:01:37.268 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:37 smithi089 ceph-mon[23733]: rocksdb: DB Session ID: CV9ZLA1IKA9NC63X5QTH 2024-09-21T04:01:37.268 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:37 smithi089 ceph-mon[23733]: rocksdb: CURRENT file: CURRENT 2024-09-21T04:01:37.268 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:37 smithi089 ceph-mon[23733]: rocksdb: IDENTITY file: IDENTITY 2024-09-21T04:01:37.268 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:37 smithi089 ceph-mon[23733]: rocksdb: MANIFEST file: MANIFEST-000009 size: 131 Bytes 2024-09-21T04:01:37.268 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:37 smithi089 ceph-mon[23733]: rocksdb: SST files in /var/lib/ceph/mon/ceph-smithi089/store.db dir, Total Num: 1, files: 000008.sst 2024-09-21T04:01:37.268 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:37 smithi089 ceph-mon[23733]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-smithi089/store.db: 000010.log size: 82715 ; 2024-09-21T04:01:37.268 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:37 smithi089 ceph-mon[23733]: rocksdb: Options.error_if_exists: 0 2024-09-21T04:01:37.268 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:37 smithi089 ceph-mon[23733]: rocksdb: Options.create_if_missing: 0 2024-09-21T04:01:37.268 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:37 smithi089 ceph-mon[23733]: rocksdb: Options.paranoid_checks: 1 2024-09-21T04:01:37.268 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:37 smithi089 ceph-mon[23733]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-09-21T04:01:37.268 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:37 smithi089 ceph-mon[23733]: rocksdb: Options.env: 0x55f3932c6860 2024-09-21T04:01:37.268 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:37 smithi089 ceph-mon[23733]: rocksdb: Options.fs: Posix File System 2024-09-21T04:01:37.268 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:37 smithi089 ceph-mon[23733]: rocksdb: Options.info_log: 0x55f395457fa0 2024-09-21T04:01:37.269 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:37 smithi089 ceph-mon[23733]: rocksdb: Options.max_file_opening_threads: 16 2024-09-21T04:01:37.269 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:37 smithi089 ceph-mon[23733]: rocksdb: Options.statistics: (nil) 2024-09-21T04:01:37.269 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:37 smithi089 ceph-mon[23733]: rocksdb: Options.use_fsync: 0 2024-09-21T04:01:37.269 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:37 smithi089 ceph-mon[23733]: rocksdb: Options.max_log_file_size: 0 2024-09-21T04:01:37.269 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:37 smithi089 ceph-mon[23733]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-09-21T04:01:37.269 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:37 smithi089 ceph-mon[23733]: rocksdb: Options.log_file_time_to_roll: 0 2024-09-21T04:01:37.269 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:37 smithi089 ceph-mon[23733]: rocksdb: Options.keep_log_file_num: 1000 2024-09-21T04:01:37.269 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:37 smithi089 ceph-mon[23733]: rocksdb: Options.recycle_log_file_num: 0 2024-09-21T04:01:37.269 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:37 smithi089 ceph-mon[23733]: rocksdb: Options.allow_fallocate: 1 2024-09-21T04:01:37.269 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:37 smithi089 ceph-mon[23733]: rocksdb: Options.allow_mmap_reads: 0 2024-09-21T04:01:37.269 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:37 smithi089 ceph-mon[23733]: rocksdb: Options.allow_mmap_writes: 0 2024-09-21T04:01:37.269 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:37 smithi089 ceph-mon[23733]: rocksdb: Options.use_direct_reads: 0 2024-09-21T04:01:37.269 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:37 smithi089 ceph-mon[23733]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-09-21T04:01:37.269 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:37 smithi089 ceph-mon[23733]: rocksdb: Options.create_missing_column_families: 0 2024-09-21T04:01:37.269 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:37 smithi089 ceph-mon[23733]: rocksdb: Options.db_log_dir: 2024-09-21T04:01:37.270 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:37 smithi089 ceph-mon[23733]: rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-smithi089/store.db 2024-09-21T04:01:37.270 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:37 smithi089 ceph-mon[23733]: rocksdb: Options.table_cache_numshardbits: 6 2024-09-21T04:01:37.270 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:37 smithi089 ceph-mon[23733]: rocksdb: Options.WAL_ttl_seconds: 0 2024-09-21T04:01:37.270 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:37 smithi089 ceph-mon[23733]: rocksdb: Options.WAL_size_limit_MB: 0 2024-09-21T04:01:37.270 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:37 smithi089 ceph-mon[23733]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-09-21T04:01:37.270 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:37 smithi089 ceph-mon[23733]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-09-21T04:01:37.270 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:37 smithi089 ceph-mon[23733]: rocksdb: Options.is_fd_close_on_exec: 1 2024-09-21T04:01:37.270 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:37 smithi089 ceph-mon[23733]: rocksdb: Options.advise_random_on_open: 1 2024-09-21T04:01:37.270 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:37 smithi089 ceph-mon[23733]: rocksdb: Options.db_write_buffer_size: 0 2024-09-21T04:01:37.270 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:37 smithi089 ceph-mon[23733]: rocksdb: Options.write_buffer_manager: 0x55f395548330 2024-09-21T04:01:37.270 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:37 smithi089 ceph-mon[23733]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-09-21T04:01:37.270 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:37 smithi089 ceph-mon[23733]: rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2024-09-21T04:01:37.270 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:37 smithi089 ceph-mon[23733]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-09-21T04:01:37.270 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:37 smithi089 ceph-mon[23733]: rocksdb: Options.use_adaptive_mutex: 0 2024-09-21T04:01:37.271 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:37 smithi089 ceph-mon[23733]: rocksdb: Options.rate_limiter: (nil) 2024-09-21T04:01:37.271 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:37 smithi089 ceph-mon[23733]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-09-21T04:01:37.271 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:37 smithi089 ceph-mon[23733]: rocksdb: Options.wal_recovery_mode: 2 2024-09-21T04:01:37.271 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:37 smithi089 ceph-mon[23733]: rocksdb: Options.enable_thread_tracking: 0 2024-09-21T04:01:37.271 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:37 smithi089 ceph-mon[23733]: rocksdb: Options.enable_pipelined_write: 0 2024-09-21T04:01:37.271 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:37 smithi089 ceph-mon[23733]: rocksdb: Options.unordered_write: 0 2024-09-21T04:01:37.271 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:37 smithi089 ceph-mon[23733]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-09-21T04:01:37.271 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:37 smithi089 ceph-mon[23733]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-09-21T04:01:37.271 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:37 smithi089 ceph-mon[23733]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-09-21T04:01:37.271 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:37 smithi089 ceph-mon[23733]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-09-21T04:01:37.271 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:37 smithi089 ceph-mon[23733]: rocksdb: Options.row_cache: None 2024-09-21T04:01:37.271 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:37 smithi089 ceph-mon[23733]: rocksdb: Options.wal_filter: None 2024-09-21T04:01:37.271 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:37 smithi089 ceph-mon[23733]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-09-21T04:01:37.271 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:37 smithi089 ceph-mon[23733]: rocksdb: Options.allow_ingest_behind: 0 2024-09-21T04:01:37.271 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:37 smithi089 ceph-mon[23733]: rocksdb: Options.preserve_deletes: 0 2024-09-21T04:01:37.272 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:37 smithi089 ceph-mon[23733]: rocksdb: Options.two_write_queues: 0 2024-09-21T04:01:37.272 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:37 smithi089 ceph-mon[23733]: rocksdb: Options.manual_wal_flush: 0 2024-09-21T04:01:37.272 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:37 smithi089 ceph-mon[23733]: rocksdb: Options.atomic_flush: 0 2024-09-21T04:01:37.272 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:37 smithi089 ceph-mon[23733]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-09-21T04:01:37.272 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:37 smithi089 ceph-mon[23733]: rocksdb: Options.persist_stats_to_disk: 0 2024-09-21T04:01:37.272 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:37 smithi089 ceph-mon[23733]: rocksdb: Options.write_dbid_to_manifest: 0 2024-09-21T04:01:37.272 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:37 smithi089 ceph-mon[23733]: rocksdb: Options.log_readahead_size: 0 2024-09-21T04:01:37.272 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:37 smithi089 ceph-mon[23733]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-09-21T04:01:37.272 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:37 smithi089 ceph-mon[23733]: rocksdb: Options.best_efforts_recovery: 0 2024-09-21T04:01:37.272 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:37 smithi089 ceph-mon[23733]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-09-21T04:01:37.272 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:37 smithi089 ceph-mon[23733]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-09-21T04:01:37.272 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:37 smithi089 ceph-mon[23733]: rocksdb: Options.allow_data_in_errors: 0 2024-09-21T04:01:37.272 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:37 smithi089 ceph-mon[23733]: rocksdb: Options.db_host_id: __hostname__ 2024-09-21T04:01:37.272 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:37 smithi089 ceph-mon[23733]: rocksdb: Options.max_background_jobs: 2 2024-09-21T04:01:37.272 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:37 smithi089 ceph-mon[23733]: rocksdb: Options.max_background_compactions: -1 2024-09-21T04:01:37.273 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:37 smithi089 ceph-mon[23733]: rocksdb: Options.max_subcompactions: 1 2024-09-21T04:01:37.273 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:37 smithi089 ceph-mon[23733]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-09-21T04:01:37.273 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:37 smithi089 ceph-mon[23733]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-09-21T04:01:37.273 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:37 smithi089 ceph-mon[23733]: rocksdb: Options.delayed_write_rate : 16777216 2024-09-21T04:01:37.273 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:37 smithi089 ceph-mon[23733]: rocksdb: Options.max_total_wal_size: 0 2024-09-21T04:01:37.273 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:37 smithi089 ceph-mon[23733]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-09-21T04:01:37.273 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:37 smithi089 ceph-mon[23733]: rocksdb: Options.stats_dump_period_sec: 600 2024-09-21T04:01:37.273 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:37 smithi089 ceph-mon[23733]: rocksdb: Options.stats_persist_period_sec: 600 2024-09-21T04:01:37.273 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:37 smithi089 ceph-mon[23733]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-09-21T04:01:37.273 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:37 smithi089 ceph-mon[23733]: rocksdb: Options.max_open_files: -1 2024-09-21T04:01:37.273 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:37 smithi089 ceph-mon[23733]: rocksdb: Options.bytes_per_sync: 0 2024-09-21T04:01:37.273 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:37 smithi089 ceph-mon[23733]: rocksdb: Options.wal_bytes_per_sync: 0 2024-09-21T04:01:37.273 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:37 smithi089 ceph-mon[23733]: rocksdb: Options.strict_bytes_per_sync: 0 2024-09-21T04:01:37.273 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:37 smithi089 ceph-mon[23733]: rocksdb: Options.compaction_readahead_size: 0 2024-09-21T04:01:37.274 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:37 smithi089 ceph-mon[23733]: rocksdb: Options.max_background_flushes: -1 2024-09-21T04:01:37.274 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:37 smithi089 ceph-mon[23733]: rocksdb: Compression algorithms supported: 2024-09-21T04:01:37.274 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:37 smithi089 ceph-mon[23733]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-09-21T04:01:37.274 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:37 smithi089 ceph-mon[23733]: rocksdb: kZSTD supported: 0 2024-09-21T04:01:37.274 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:37 smithi089 ceph-mon[23733]: rocksdb: kXpressCompression supported: 0 2024-09-21T04:01:37.274 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:37 smithi089 ceph-mon[23733]: rocksdb: kLZ4HCCompression supported: 1 2024-09-21T04:01:37.274 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:37 smithi089 ceph-mon[23733]: rocksdb: kLZ4Compression supported: 1 2024-09-21T04:01:37.274 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:37 smithi089 ceph-mon[23733]: rocksdb: kBZip2Compression supported: 0 2024-09-21T04:01:37.274 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:37 smithi089 ceph-mon[23733]: rocksdb: kZlibCompression supported: 1 2024-09-21T04:01:37.274 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:37 smithi089 ceph-mon[23733]: rocksdb: kSnappyCompression supported: 1 2024-09-21T04:01:37.274 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:37 smithi089 ceph-mon[23733]: rocksdb: Fast CRC32 supported: Supported on x86 2024-09-21T04:01:37.274 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:37 smithi089 ceph-mon[23733]: rocksdb: [db/version_set.cc:4725] Recovering from manifest file: /var/lib/ceph/mon/ceph-smithi089/store.db/MANIFEST-000009 2024-09-21T04:01:37.274 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:37 smithi089 ceph-mon[23733]: rocksdb: [db/column_family.cc:597] --------------- Options for column family [default]: 2024-09-21T04:01:37.274 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:37 smithi089 ceph-mon[23733]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-09-21T04:01:37.274 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:37 smithi089 ceph-mon[23733]: rocksdb: Options.merge_operator: 2024-09-21T04:01:37.275 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:37 smithi089 ceph-mon[23733]: rocksdb: Options.compaction_filter: None 2024-09-21T04:01:37.275 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:37 smithi089 ceph-mon[23733]: rocksdb: Options.compaction_filter_factory: None 2024-09-21T04:01:37.275 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:37 smithi089 ceph-mon[23733]: rocksdb: Options.sst_partitioner_factory: None 2024-09-21T04:01:37.275 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:37 smithi089 ceph-mon[23733]: rocksdb: Options.memtable_factory: SkipListFactory 2024-09-21T04:01:37.275 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:37 smithi089 ceph-mon[23733]: rocksdb: Options.table_factory: BlockBasedTable 2024-09-21T04:01:37.275 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:37 smithi089 ceph-mon[23733]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55f395423d00) 2024-09-21T04:01:37.275 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout: cache_index_and_filter_blocks: 1 2024-09-21T04:01:37.275 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-09-21T04:01:37.275 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-09-21T04:01:37.275 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout: pin_top_level_index_and_filter: 1 2024-09-21T04:01:37.275 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout: index_type: 0 2024-09-21T04:01:37.275 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout: data_block_index_type: 0 2024-09-21T04:01:37.275 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout: index_shortening: 1 2024-09-21T04:01:37.275 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout: data_block_hash_table_util_ratio: 0.750000 2024-09-21T04:01:37.275 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout: hash_index_allow_collision: 1 2024-09-21T04:01:37.276 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout: checksum: 1 2024-09-21T04:01:37.276 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout: no_block_cache: 0 2024-09-21T04:01:37.276 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout: block_cache: 0x55f39548e170 2024-09-21T04:01:37.276 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout: block_cache_name: BinnedLRUCache 2024-09-21T04:01:37.276 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout: block_cache_options: 2024-09-21T04:01:37.276 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout: capacity : 536870912 2024-09-21T04:01:37.276 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout: num_shard_bits : 4 2024-09-21T04:01:37.276 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout: strict_capacity_limit : 0 2024-09-21T04:01:37.276 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout: high_pri_pool_ratio: 0.000 2024-09-21T04:01:37.276 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout: block_cache_compressed: (nil) 2024-09-21T04:01:37.276 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout: persistent_cache: (nil) 2024-09-21T04:01:37.276 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout: block_size: 4096 2024-09-21T04:01:37.276 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout: block_size_deviation: 10 2024-09-21T04:01:37.276 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout: block_restart_interval: 16 2024-09-21T04:01:37.276 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout: index_block_restart_interval: 1 2024-09-21T04:01:37.277 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout: metadata_block_size: 4096 2024-09-21T04:01:37.277 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout: partition_filters: 0 2024-09-21T04:01:37.277 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout: use_delta_encoding: 1 2024-09-21T04:01:37.277 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout: filter_policy: rocksdb.BuiltinBloomFilter 2024-09-21T04:01:37.277 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout: whole_key_filtering: 1 2024-09-21T04:01:37.277 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout: verify_compression: 0 2024-09-21T04:01:37.277 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout: read_amp_bytes_per_bit: 0 2024-09-21T04:01:37.277 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout: format_version: 4 2024-09-21T04:01:37.277 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout: enable_index_compression: 1 2024-09-21T04:01:37.277 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout: block_align: 0 2024-09-21T04:01:37.277 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:37 smithi089 ceph-mon[23733]: rocksdb: Options.write_buffer_size: 33554432 2024-09-21T04:01:37.277 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:37 smithi089 ceph-mon[23733]: rocksdb: Options.max_write_buffer_number: 2 2024-09-21T04:01:37.277 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:37 smithi089 ceph-mon[23733]: rocksdb: Options.compression: NoCompression 2024-09-21T04:01:37.277 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:37 smithi089 ceph-mon[23733]: rocksdb: Options.bottommost_compression: Disabled 2024-09-21T04:01:37.277 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:37 smithi089 ceph-mon[23733]: rocksdb: Options.prefix_extractor: nullptr 2024-09-21T04:01:37.277 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:37 smithi089 ceph-mon[23733]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-09-21T04:01:37.278 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:37 smithi089 ceph-mon[23733]: rocksdb: Options.num_levels: 7 2024-09-21T04:01:37.278 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:37 smithi089 ceph-mon[23733]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-09-21T04:01:37.278 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:37 smithi089 ceph-mon[23733]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-09-21T04:01:37.278 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:37 smithi089 ceph-mon[23733]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-09-21T04:01:37.278 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:37 smithi089 ceph-mon[23733]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-09-21T04:01:37.278 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:37 smithi089 ceph-mon[23733]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-09-21T04:01:37.278 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:37 smithi089 ceph-mon[23733]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-09-21T04:01:37.278 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:37 smithi089 ceph-mon[23733]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-09-21T04:01:37.278 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:37 smithi089 ceph-mon[23733]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-09-21T04:01:37.278 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:37 smithi089 ceph-mon[23733]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-09-21T04:01:37.278 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:37 smithi089 ceph-mon[23733]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-09-21T04:01:37.278 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:37 smithi089 ceph-mon[23733]: rocksdb: Options.compression_opts.window_bits: -14 2024-09-21T04:01:37.278 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:37 smithi089 ceph-mon[23733]: rocksdb: Options.compression_opts.level: 32767 2024-09-21T04:01:37.278 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:37 smithi089 ceph-mon[23733]: rocksdb: Options.compression_opts.strategy: 0 2024-09-21T04:01:37.279 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:37 smithi089 ceph-mon[23733]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-09-21T04:01:37.279 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:37 smithi089 ceph-mon[23733]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-09-21T04:01:37.279 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:37 smithi089 ceph-mon[23733]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-09-21T04:01:37.279 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:37 smithi089 ceph-mon[23733]: rocksdb: Options.compression_opts.enabled: false 2024-09-21T04:01:37.279 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:37 smithi089 ceph-mon[23733]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-09-21T04:01:37.279 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:37 smithi089 ceph-mon[23733]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-09-21T04:01:37.279 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:37 smithi089 ceph-mon[23733]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-09-21T04:01:37.279 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:37 smithi089 ceph-mon[23733]: rocksdb: Options.target_file_size_base: 67108864 2024-09-21T04:01:37.279 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:37 smithi089 ceph-mon[23733]: rocksdb: Options.target_file_size_multiplier: 1 2024-09-21T04:01:37.279 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:37 smithi089 ceph-mon[23733]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-09-21T04:01:37.279 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:37 smithi089 ceph-mon[23733]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-09-21T04:01:37.279 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:37 smithi089 ceph-mon[23733]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-09-21T04:01:37.279 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:37 smithi089 ceph-mon[23733]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-09-21T04:01:37.279 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:37 smithi089 ceph-mon[23733]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-09-21T04:01:37.279 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:37 smithi089 ceph-mon[23733]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-09-21T04:01:37.279 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:37 smithi089 ceph-mon[23733]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-09-21T04:01:37.280 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:37 smithi089 ceph-mon[23733]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-09-21T04:01:37.280 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:37 smithi089 ceph-mon[23733]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-09-21T04:01:37.280 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:37 smithi089 ceph-mon[23733]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-09-21T04:01:37.280 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:37 smithi089 ceph-mon[23733]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-09-21T04:01:37.280 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:37 smithi089 ceph-mon[23733]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-09-21T04:01:37.280 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:37 smithi089 ceph-mon[23733]: rocksdb: Options.arena_block_size: 4194304 2024-09-21T04:01:37.280 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:37 smithi089 ceph-mon[23733]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-09-21T04:01:37.280 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:37 smithi089 ceph-mon[23733]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-09-21T04:01:37.280 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:37 smithi089 ceph-mon[23733]: rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2024-09-21T04:01:37.280 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:37 smithi089 ceph-mon[23733]: rocksdb: Options.disable_auto_compactions: 0 2024-09-21T04:01:37.280 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:37 smithi089 ceph-mon[23733]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-09-21T04:01:37.280 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:37 smithi089 ceph-mon[23733]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-09-21T04:01:37.280 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:37 smithi089 ceph-mon[23733]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-09-21T04:01:37.280 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:37 smithi089 ceph-mon[23733]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-09-21T04:01:37.280 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:37 smithi089 ceph-mon[23733]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-09-21T04:01:37.281 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:37 smithi089 ceph-mon[23733]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-09-21T04:01:37.281 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:37 smithi089 ceph-mon[23733]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-09-21T04:01:37.281 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:37 smithi089 ceph-mon[23733]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-09-21T04:01:37.281 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:37 smithi089 ceph-mon[23733]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-09-21T04:01:37.281 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:37 smithi089 ceph-mon[23733]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-09-21T04:01:37.281 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:37 smithi089 ceph-mon[23733]: rocksdb: Options.table_properties_collectors: 2024-09-21T04:01:37.281 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:37 smithi089 ceph-mon[23733]: rocksdb: Options.inplace_update_support: 0 2024-09-21T04:01:37.281 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:37 smithi089 ceph-mon[23733]: rocksdb: Options.inplace_update_num_locks: 10000 2024-09-21T04:01:37.281 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:37 smithi089 ceph-mon[23733]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-09-21T04:01:37.281 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:37 smithi089 ceph-mon[23733]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-09-21T04:01:37.281 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:37 smithi089 ceph-mon[23733]: rocksdb: Options.memtable_huge_page_size: 0 2024-09-21T04:01:37.281 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:37 smithi089 ceph-mon[23733]: rocksdb: Options.bloom_locality: 0 2024-09-21T04:01:37.281 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:37 smithi089 ceph-mon[23733]: rocksdb: Options.max_successive_merges: 0 2024-09-21T04:01:37.281 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:37 smithi089 ceph-mon[23733]: rocksdb: Options.optimize_filters_for_hits: 0 2024-09-21T04:01:37.281 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:37 smithi089 ceph-mon[23733]: rocksdb: Options.paranoid_file_checks: 0 2024-09-21T04:01:37.282 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:37 smithi089 ceph-mon[23733]: rocksdb: Options.force_consistency_checks: 1 2024-09-21T04:01:37.282 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:37 smithi089 ceph-mon[23733]: rocksdb: Options.report_bg_io_stats: 0 2024-09-21T04:01:37.282 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:37 smithi089 ceph-mon[23733]: rocksdb: Options.ttl: 2592000 2024-09-21T04:01:37.282 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:37 smithi089 ceph-mon[23733]: rocksdb: Options.periodic_compaction_seconds: 0 2024-09-21T04:01:37.282 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:37 smithi089 ceph-mon[23733]: rocksdb: Options.enable_blob_files: false 2024-09-21T04:01:37.282 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:37 smithi089 ceph-mon[23733]: rocksdb: Options.min_blob_size: 0 2024-09-21T04:01:37.282 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:37 smithi089 ceph-mon[23733]: rocksdb: Options.blob_file_size: 268435456 2024-09-21T04:01:37.282 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:37 smithi089 ceph-mon[23733]: rocksdb: Options.blob_compression_type: NoCompression 2024-09-21T04:01:37.282 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:37 smithi089 ceph-mon[23733]: rocksdb: Options.enable_blob_garbage_collection: false 2024-09-21T04:01:37.282 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:37 smithi089 ceph-mon[23733]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-09-21T04:01:37.282 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:37 smithi089 ceph-mon[23733]: rocksdb: [db/version_set.cc:4773] Recovered from manifest file:/var/lib/ceph/mon/ceph-smithi089/store.db/MANIFEST-000009 succeeded,manifest_file_number is 9, next_file_number is 11, last_sequence is 5, log_number is 5,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-09-21T04:01:37.282 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:37 smithi089 ceph-mon[23733]: rocksdb: [db/version_set.cc:4782] Column family [default] (ID 0), log number is 5 2024-09-21T04:01:37.282 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:37 smithi089 ceph-mon[23733]: rocksdb: [db/version_set.cc:4083] Creating manifest 13 2024-09-21T04:01:37.282 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:37 smithi089 ceph-mon[23733]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726891297039708, "job": 1, "event": "recovery_started", "wal_files": [10]} 2024-09-21T04:01:37.282 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:37 smithi089 ceph-mon[23733]: rocksdb: [db/db_impl/db_impl_open.cc:847] Recovering log #10 mode 2 2024-09-21T04:01:37.283 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:37 smithi089 ceph-mon[23733]: rocksdb: [table/block_based/filter_policy.cc:996] Using legacy Bloom filter with high (20) bits/key. Dramatic filter space and/or accuracy improvement is available with format_version>=5. 2024-09-21T04:01:37.283 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:37 smithi089 ceph-mon[23733]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726891297040695, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 14, "file_size": 79397, "file_checksum": "", "file_checksum_func_name": "Unknown", "table_properties": {"data_size": 77661, "index_size": 224, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 581, "raw_key_size": 9884, "raw_average_key_size": 47, "raw_value_size": 71970, "raw_average_value_size": 344, "num_data_blocks": 10, "num_entries": 209, "num_deletions": 3, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "rocksdb.BuiltinBloomFilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; ", "creation_time": 1726891297, "oldest_key_time": 0, "file_creation_time": 0, "db_id": "ec7c5a60-1c31-4106-9df1-609690964e8c", "db_session_id": "CV9ZLA1IKA9NC63X5QTH"}} 2024-09-21T04:01:37.283 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:37 smithi089 ceph-mon[23733]: rocksdb: [db/version_set.cc:4083] Creating manifest 15 2024-09-21T04:01:37.283 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:37 smithi089 ceph-mon[23733]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726891297041115, "job": 1, "event": "recovery_finished"} 2024-09-21T04:01:37.283 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:37 smithi089 ceph-mon[23733]: rocksdb: [file/delete_scheduler.cc:73] Deleted file /var/lib/ceph/mon/ceph-smithi089/store.db/000010.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-09-21T04:01:37.283 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:37 smithi089 ceph-mon[23733]: rocksdb: [db/db_impl/db_impl_open.cc:1701] SstFileManager instance 0x55f395474a80 2024-09-21T04:01:37.283 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:37 smithi089 ceph-mon[23733]: rocksdb: DB pointer 0x55f395484000 2024-09-21T04:01:37.283 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:37 smithi089 ceph-mon[23733]: rocksdb: [db/db_impl/db_impl.cc:902] ------- DUMPING STATS ------- 2024-09-21T04:01:37.283 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:37 smithi089 ceph-mon[23733]: rocksdb: [db/db_impl/db_impl.cc:903] 2024-09-21T04:01:37.283 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout: ** DB Stats ** 2024-09-21T04:01:37.283 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-21T04:01:37.283 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-09-21T04:01:37.283 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-21T04:01:37.283 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-21T04:01:37.283 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-09-21T04:01:37.284 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2024-09-21T04:01:37.284 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-21T04:01:37.284 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout: 2024-09-21T04:01:37.284 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout: ** Compaction Stats [default] ** 2024-09-21T04:01:37.284 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-21T04:01:37.284 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-21T04:01:37.284 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout: L0 2/0 79.33 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 117.0 0.00 0.00 1 0.001 0 0 2024-09-21T04:01:37.284 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout: Sum 2/0 79.33 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 117.0 0.00 0.00 1 0.001 0 0 2024-09-21T04:01:37.284 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 117.0 0.00 0.00 1 0.001 0 0 2024-09-21T04:01:37.284 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout: 2024-09-21T04:01:37.284 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout: ** Compaction Stats [default] ** 2024-09-21T04:01:37.284 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-21T04:01:37.284 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-21T04:01:37.284 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 117.0 0.00 0.00 1 0.001 0 0 2024-09-21T04:01:37.284 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-21T04:01:37.285 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-09-21T04:01:37.285 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-09-21T04:01:37.285 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-09-21T04:01:37.285 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-09-21T04:01:37.285 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout: AddFile(Keys): cumulative 0, interval 0 2024-09-21T04:01:37.285 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout: Cumulative compaction: 0.00 GB write, 18.46 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-21T04:01:37.285 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout: Interval compaction: 0.00 GB write, 18.46 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-21T04:01:37.285 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-09-21T04:01:37.285 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout: 2024-09-21T04:01:37.285 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout: ** File Read Latency Histogram By Level [default] ** 2024-09-21T04:01:37.285 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout: 2024-09-21T04:01:37.285 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout: ** Compaction Stats [default] ** 2024-09-21T04:01:37.285 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-21T04:01:37.285 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-21T04:01:37.286 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout: L0 2/0 79.33 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 117.0 0.00 0.00 1 0.001 0 0 2024-09-21T04:01:37.286 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout: Sum 2/0 79.33 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 117.0 0.00 0.00 1 0.001 0 0 2024-09-21T04:01:37.286 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2024-09-21T04:01:37.286 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout: 2024-09-21T04:01:37.286 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout: ** Compaction Stats [default] ** 2024-09-21T04:01:37.286 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-21T04:01:37.286 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-21T04:01:37.286 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 117.0 0.00 0.00 1 0.001 0 0 2024-09-21T04:01:37.286 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-21T04:01:37.286 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-09-21T04:01:37.286 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-09-21T04:01:37.286 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-09-21T04:01:37.286 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-09-21T04:01:37.286 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout: AddFile(Keys): cumulative 0, interval 0 2024-09-21T04:01:37.286 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout: Cumulative compaction: 0.00 GB write, 18.33 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-21T04:01:37.287 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-21T04:01:37.287 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-09-21T04:01:37.287 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout: 2024-09-21T04:01:37.287 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout: ** File Read Latency Histogram By Level [default] ** 2024-09-21T04:01:37.287 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:37 smithi089 ceph-mon[23733]: starting mon.smithi089 rank 0 at public addrs [v2:172.21.15.89:3300/0,v1:172.21.15.89:6789/0] at bind addrs [v2:172.21.15.89:3300/0,v1:172.21.15.89:6789/0] mon_data /var/lib/ceph/mon/ceph-smithi089 fsid 1b9b79fc-77ce-11ef-baf5-efdc52797490 2024-09-21T04:01:37.287 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:37 smithi089 ceph-mon[23733]: mon.smithi089@-1(???) e1 preinit fsid 1b9b79fc-77ce-11ef-baf5-efdc52797490 2024-09-21T04:01:37.287 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:37 smithi089 ceph-mon[23733]: mon.smithi089@-1(???).mds e1 new map 2024-09-21T04:01:37.287 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:37 smithi089 ceph-mon[23733]: mon.smithi089@-1(???).mds e1 print_map 2024-09-21T04:01:37.287 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout: e1 2024-09-21T04:01:37.287 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-09-21T04:01:37.287 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-09-21T04:01:37.287 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout: legacy client fscid: -1 2024-09-21T04:01:37.287 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout: 2024-09-21T04:01:37.287 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout: No filesystems configured 2024-09-21T04:01:37.287 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:37 smithi089 ceph-mon[23733]: mon.smithi089@-1(???).osd e1 crush map has features 3314932999778484224, adjusting msgr requires 2024-09-21T04:01:37.288 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:37 smithi089 ceph-mon[23733]: mon.smithi089@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-09-21T04:01:37.288 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:37 smithi089 ceph-mon[23733]: mon.smithi089@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-09-21T04:01:37.288 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:37 smithi089 ceph-mon[23733]: mon.smithi089@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-09-21T04:01:37.288 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:37 smithi089 ceph-mon[23733]: mon.smithi089@-1(???).paxosservice(auth 1..2) refresh upgraded, format 0 -> 3 2024-09-21T04:01:37.288 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:37 smithi089 ceph-mon[23733]: mon.smithi089 is new leader, mons smithi089 in quorum (ranks 0) 2024-09-21T04:01:37.288 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:37 smithi089 ceph-mon[23733]: monmap e1: 1 mons at {smithi089=[v2:172.21.15.89:3300/0,v1:172.21.15.89:6789/0]} 2024-09-21T04:01:37.288 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:37 smithi089 ceph-mon[23733]: fsmap 2024-09-21T04:01:37.288 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:37 smithi089 ceph-mon[23733]: osdmap e1: 0 total, 0 up, 0 in 2024-09-21T04:01:37.288 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:37 smithi089 ceph-mon[23733]: mgrmap e1: no daemons active 2024-09-21T04:01:37.288 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:37 smithi089 bash[23715]: 1d2b3719e46efeb28714f3f5bdfa041021ff4c3c0f21316359a357c8b7c7b7d9 2024-09-21T04:01:37.288 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:37 smithi089 systemd[1]: Started Ceph mon.smithi089 for 1b9b79fc-77ce-11ef-baf5-efdc52797490. 2024-09-21T04:01:38.110 INFO:teuthology.orchestra.run.smithi089.stderr:Wrote config to /etc/ceph/ceph.conf 2024-09-21T04:01:38.110 INFO:teuthology.orchestra.run.smithi089.stderr:Wrote keyring to /etc/ceph/ceph.client.admin.keyring 2024-09-21T04:01:38.110 INFO:teuthology.orchestra.run.smithi089.stderr:Creating mgr... 2024-09-21T04:01:38.111 INFO:teuthology.orchestra.run.smithi089.stderr:Verifying port 9283 ... 2024-09-21T04:01:38.287 INFO:teuthology.orchestra.run.smithi089.stderr:systemctl: Failed to reset failed state of unit ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490@mgr.smithi089.srlyxf.service: Unit ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490@mgr.smithi089.srlyxf.service not loaded. 2024-09-21T04:01:38.300 INFO:teuthology.orchestra.run.smithi089.stderr:systemctl: Created symlink /etc/systemd/system/ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490.target.wants/ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490@mgr.smithi089.srlyxf.service → /etc/systemd/system/ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490@.service. 2024-09-21T04:01:38.765 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:38 smithi089 ceph-mon[23733]: from='client.? 172.21.15.89:0/3804477749' entity='client.admin' 2024-09-21T04:01:39.012 INFO:teuthology.orchestra.run.smithi089.stderr:systemctl: disabled 2024-09-21T04:01:39.019 INFO:teuthology.orchestra.run.smithi089.stderr:systemctl: inactive 2024-09-21T04:01:39.020 INFO:teuthology.orchestra.run.smithi089.stderr:firewalld.service is not enabled 2024-09-21T04:01:39.020 INFO:teuthology.orchestra.run.smithi089.stderr:Not possible to enable service . firewalld.service is not available 2024-09-21T04:01:39.028 INFO:teuthology.orchestra.run.smithi089.stderr:systemctl: disabled 2024-09-21T04:01:39.036 INFO:teuthology.orchestra.run.smithi089.stderr:systemctl: inactive 2024-09-21T04:01:39.036 INFO:teuthology.orchestra.run.smithi089.stderr:firewalld.service is not enabled 2024-09-21T04:01:39.036 INFO:teuthology.orchestra.run.smithi089.stderr:Not possible to open ports <[9283]>. firewalld.service is not available 2024-09-21T04:01:39.036 INFO:teuthology.orchestra.run.smithi089.stderr:Waiting for mgr to start... 2024-09-21T04:01:39.037 INFO:teuthology.orchestra.run.smithi089.stderr:Waiting for mgr... 2024-09-21T04:01:39.549 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: 2024-09-21T04:01:39.549 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: { 2024-09-21T04:01:39.549 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "fsid": "1b9b79fc-77ce-11ef-baf5-efdc52797490", 2024-09-21T04:01:39.549 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "health": { 2024-09-21T04:01:39.549 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "status": "HEALTH_OK", 2024-09-21T04:01:39.549 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "checks": {}, 2024-09-21T04:01:39.549 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "mutes": [] 2024-09-21T04:01:39.550 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: }, 2024-09-21T04:01:39.550 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "election_epoch": 5, 2024-09-21T04:01:39.550 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "quorum": [ 2024-09-21T04:01:39.550 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: 0 2024-09-21T04:01:39.550 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: ], 2024-09-21T04:01:39.550 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "quorum_names": [ 2024-09-21T04:01:39.550 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "smithi089" 2024-09-21T04:01:39.550 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: ], 2024-09-21T04:01:39.550 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "quorum_age": 2, 2024-09-21T04:01:39.550 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "monmap": { 2024-09-21T04:01:39.550 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-21T04:01:39.550 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "min_mon_release_name": "quincy", 2024-09-21T04:01:39.550 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "num_mons": 1 2024-09-21T04:01:39.550 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: }, 2024-09-21T04:01:39.551 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "osdmap": { 2024-09-21T04:01:39.551 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-21T04:01:39.551 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "num_osds": 0, 2024-09-21T04:01:39.551 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "num_up_osds": 0, 2024-09-21T04:01:39.551 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "osd_up_since": 0, 2024-09-21T04:01:39.551 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "num_in_osds": 0, 2024-09-21T04:01:39.551 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "osd_in_since": 0, 2024-09-21T04:01:39.551 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "num_remapped_pgs": 0 2024-09-21T04:01:39.551 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: }, 2024-09-21T04:01:39.551 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "pgmap": { 2024-09-21T04:01:39.551 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "pgs_by_state": [], 2024-09-21T04:01:39.551 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "num_pgs": 0, 2024-09-21T04:01:39.551 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "num_pools": 0, 2024-09-21T04:01:39.551 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "num_objects": 0, 2024-09-21T04:01:39.551 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "data_bytes": 0, 2024-09-21T04:01:39.552 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "bytes_used": 0, 2024-09-21T04:01:39.552 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "bytes_avail": 0, 2024-09-21T04:01:39.552 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "bytes_total": 0 2024-09-21T04:01:39.552 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: }, 2024-09-21T04:01:39.552 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "fsmap": { 2024-09-21T04:01:39.552 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-21T04:01:39.552 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "by_rank": [], 2024-09-21T04:01:39.552 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "up:standby": 0 2024-09-21T04:01:39.552 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: }, 2024-09-21T04:01:39.552 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "mgrmap": { 2024-09-21T04:01:39.552 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "available": false, 2024-09-21T04:01:39.552 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "num_standbys": 0, 2024-09-21T04:01:39.552 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "modules": [ 2024-09-21T04:01:39.552 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "iostat", 2024-09-21T04:01:39.552 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "nfs", 2024-09-21T04:01:39.553 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "restful" 2024-09-21T04:01:39.553 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: ], 2024-09-21T04:01:39.553 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "services": {} 2024-09-21T04:01:39.553 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: }, 2024-09-21T04:01:39.553 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "servicemap": { 2024-09-21T04:01:39.553 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-21T04:01:39.553 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "modified": "2024-09-21T04:01:32.259637+0000", 2024-09-21T04:01:39.553 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "services": {} 2024-09-21T04:01:39.553 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: }, 2024-09-21T04:01:39.553 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "progress_events": {} 2024-09-21T04:01:39.553 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: } 2024-09-21T04:01:39.889 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:39 smithi089 ceph-mon[23733]: from='client.? 172.21.15.89:0/3981198116' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-09-21T04:01:40.816 INFO:teuthology.orchestra.run.smithi089.stderr:mgr not available, waiting (1/15)... 2024-09-21T04:01:43.295 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: 2024-09-21T04:01:43.295 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: { 2024-09-21T04:01:43.295 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "fsid": "1b9b79fc-77ce-11ef-baf5-efdc52797490", 2024-09-21T04:01:43.295 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "health": { 2024-09-21T04:01:43.295 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "status": "HEALTH_OK", 2024-09-21T04:01:43.295 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "checks": {}, 2024-09-21T04:01:43.295 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "mutes": [] 2024-09-21T04:01:43.296 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: }, 2024-09-21T04:01:43.296 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "election_epoch": 5, 2024-09-21T04:01:43.296 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "quorum": [ 2024-09-21T04:01:43.296 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: 0 2024-09-21T04:01:43.296 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: ], 2024-09-21T04:01:43.296 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "quorum_names": [ 2024-09-21T04:01:43.296 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "smithi089" 2024-09-21T04:01:43.296 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: ], 2024-09-21T04:01:43.296 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "quorum_age": 6, 2024-09-21T04:01:43.296 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "monmap": { 2024-09-21T04:01:43.296 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-21T04:01:43.296 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "min_mon_release_name": "quincy", 2024-09-21T04:01:43.296 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "num_mons": 1 2024-09-21T04:01:43.296 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: }, 2024-09-21T04:01:43.296 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "osdmap": { 2024-09-21T04:01:43.297 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-21T04:01:43.297 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "num_osds": 0, 2024-09-21T04:01:43.297 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "num_up_osds": 0, 2024-09-21T04:01:43.297 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "osd_up_since": 0, 2024-09-21T04:01:43.297 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "num_in_osds": 0, 2024-09-21T04:01:43.300 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "osd_in_since": 0, 2024-09-21T04:01:43.301 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "num_remapped_pgs": 0 2024-09-21T04:01:43.301 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: }, 2024-09-21T04:01:43.301 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "pgmap": { 2024-09-21T04:01:43.301 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "pgs_by_state": [], 2024-09-21T04:01:43.301 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "num_pgs": 0, 2024-09-21T04:01:43.301 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "num_pools": 0, 2024-09-21T04:01:43.301 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "num_objects": 0, 2024-09-21T04:01:43.301 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "data_bytes": 0, 2024-09-21T04:01:43.301 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "bytes_used": 0, 2024-09-21T04:01:43.301 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "bytes_avail": 0, 2024-09-21T04:01:43.301 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "bytes_total": 0 2024-09-21T04:01:43.301 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: }, 2024-09-21T04:01:43.301 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "fsmap": { 2024-09-21T04:01:43.302 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-21T04:01:43.302 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "by_rank": [], 2024-09-21T04:01:43.302 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "up:standby": 0 2024-09-21T04:01:43.302 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: }, 2024-09-21T04:01:43.302 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "mgrmap": { 2024-09-21T04:01:43.302 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "available": false, 2024-09-21T04:01:43.302 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "num_standbys": 0, 2024-09-21T04:01:43.302 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "modules": [ 2024-09-21T04:01:43.302 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "iostat", 2024-09-21T04:01:43.302 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "nfs", 2024-09-21T04:01:43.302 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "restful" 2024-09-21T04:01:43.302 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: ], 2024-09-21T04:01:43.302 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "services": {} 2024-09-21T04:01:43.302 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: }, 2024-09-21T04:01:43.302 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "servicemap": { 2024-09-21T04:01:43.303 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-21T04:01:43.303 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "modified": "2024-09-21T04:01:32.259637+0000", 2024-09-21T04:01:43.303 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "services": {} 2024-09-21T04:01:43.303 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: }, 2024-09-21T04:01:43.303 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "progress_events": {} 2024-09-21T04:01:43.303 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: } 2024-09-21T04:01:43.514 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:43 smithi089 ceph-mon[23733]: from='client.? 172.21.15.89:0/1355416421' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-09-21T04:01:43.925 INFO:teuthology.orchestra.run.smithi089.stderr:mgr not available, waiting (2/15)... 2024-09-21T04:01:45.264 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:45 smithi089 ceph-mon[23733]: Activating manager daemon smithi089.srlyxf 2024-09-21T04:01:45.264 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:45 smithi089 ceph-mon[23733]: mgrmap e2: smithi089.srlyxf(active, starting, since 0.00274144s) 2024-09-21T04:01:45.264 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:45 smithi089 ceph-mon[23733]: from='mgr.14100 172.21.15.89:0/2871216615' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-21T04:01:45.265 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:45 smithi089 ceph-mon[23733]: from='mgr.14100 172.21.15.89:0/2871216615' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-21T04:01:45.265 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:45 smithi089 ceph-mon[23733]: from='mgr.14100 172.21.15.89:0/2871216615' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-21T04:01:45.265 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:45 smithi089 ceph-mon[23733]: from='mgr.14100 172.21.15.89:0/2871216615' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "mon metadata", "id": "smithi089"}]: dispatch 2024-09-21T04:01:45.265 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:45 smithi089 ceph-mon[23733]: from='mgr.14100 172.21.15.89:0/2871216615' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "mgr metadata", "who": "smithi089.srlyxf", "id": "smithi089.srlyxf"}]: dispatch 2024-09-21T04:01:45.265 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:45 smithi089 ceph-mon[23733]: Manager daemon smithi089.srlyxf is now available 2024-09-21T04:01:45.265 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:45 smithi089 ceph-mon[23733]: from='mgr.14100 172.21.15.89:0/2871216615' entity='mgr.smithi089.srlyxf' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi089.srlyxf/mirror_snapshot_schedule"}]: dispatch 2024-09-21T04:01:45.265 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:45 smithi089 ceph-mon[23733]: from='mgr.14100 172.21.15.89:0/2871216615' entity='mgr.smithi089.srlyxf' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi089.srlyxf/trash_purge_schedule"}]: dispatch 2024-09-21T04:01:45.265 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:45 smithi089 ceph-mon[23733]: from='mgr.14100 172.21.15.89:0/2871216615' entity='mgr.smithi089.srlyxf' 2024-09-21T04:01:45.265 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:45 smithi089 ceph-mon[23733]: from='mgr.14100 172.21.15.89:0/2871216615' entity='mgr.smithi089.srlyxf' 2024-09-21T04:01:45.265 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:45 smithi089 ceph-mon[23733]: from='mgr.14100 172.21.15.89:0/2871216615' entity='mgr.smithi089.srlyxf' 2024-09-21T04:01:46.608 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: 2024-09-21T04:01:46.608 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: { 2024-09-21T04:01:46.608 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "fsid": "1b9b79fc-77ce-11ef-baf5-efdc52797490", 2024-09-21T04:01:46.608 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "health": { 2024-09-21T04:01:46.608 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "status": "HEALTH_OK", 2024-09-21T04:01:46.608 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "checks": {}, 2024-09-21T04:01:46.608 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "mutes": [] 2024-09-21T04:01:46.608 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: }, 2024-09-21T04:01:46.608 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "election_epoch": 5, 2024-09-21T04:01:46.609 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "quorum": [ 2024-09-21T04:01:46.609 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: 0 2024-09-21T04:01:46.609 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: ], 2024-09-21T04:01:46.609 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "quorum_names": [ 2024-09-21T04:01:46.609 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "smithi089" 2024-09-21T04:01:46.609 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: ], 2024-09-21T04:01:46.609 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "quorum_age": 9, 2024-09-21T04:01:46.609 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "monmap": { 2024-09-21T04:01:46.609 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-21T04:01:46.609 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "min_mon_release_name": "quincy", 2024-09-21T04:01:46.609 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "num_mons": 1 2024-09-21T04:01:46.609 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: }, 2024-09-21T04:01:46.609 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "osdmap": { 2024-09-21T04:01:46.609 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-21T04:01:46.609 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "num_osds": 0, 2024-09-21T04:01:46.610 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "num_up_osds": 0, 2024-09-21T04:01:46.610 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "osd_up_since": 0, 2024-09-21T04:01:46.613 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "num_in_osds": 0, 2024-09-21T04:01:46.613 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "osd_in_since": 0, 2024-09-21T04:01:46.613 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "num_remapped_pgs": 0 2024-09-21T04:01:46.613 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: }, 2024-09-21T04:01:46.613 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "pgmap": { 2024-09-21T04:01:46.613 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "pgs_by_state": [], 2024-09-21T04:01:46.613 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "num_pgs": 0, 2024-09-21T04:01:46.613 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "num_pools": 0, 2024-09-21T04:01:46.613 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "num_objects": 0, 2024-09-21T04:01:46.614 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "data_bytes": 0, 2024-09-21T04:01:46.614 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "bytes_used": 0, 2024-09-21T04:01:46.614 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "bytes_avail": 0, 2024-09-21T04:01:46.614 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "bytes_total": 0 2024-09-21T04:01:46.614 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: }, 2024-09-21T04:01:46.614 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "fsmap": { 2024-09-21T04:01:46.614 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-21T04:01:46.614 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "by_rank": [], 2024-09-21T04:01:46.614 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "up:standby": 0 2024-09-21T04:01:46.614 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: }, 2024-09-21T04:01:46.614 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "mgrmap": { 2024-09-21T04:01:46.614 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "available": true, 2024-09-21T04:01:46.614 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "num_standbys": 0, 2024-09-21T04:01:46.614 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "modules": [ 2024-09-21T04:01:46.614 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "iostat", 2024-09-21T04:01:46.615 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "nfs", 2024-09-21T04:01:46.615 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "restful" 2024-09-21T04:01:46.615 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: ], 2024-09-21T04:01:46.615 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "services": {} 2024-09-21T04:01:46.615 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: }, 2024-09-21T04:01:46.615 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "servicemap": { 2024-09-21T04:01:46.615 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-21T04:01:46.615 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "modified": "2024-09-21T04:01:32.259637+0000", 2024-09-21T04:01:46.615 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "services": {} 2024-09-21T04:01:46.615 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: }, 2024-09-21T04:01:46.615 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "progress_events": {} 2024-09-21T04:01:46.615 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: } 2024-09-21T04:01:47.148 INFO:teuthology.orchestra.run.smithi089.stderr:mgr is available 2024-09-21T04:01:47.247 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:46 smithi089 ceph-mon[23733]: mgrmap e3: smithi089.srlyxf(active, since 1.00595s) 2024-09-21T04:01:47.247 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:46 smithi089 ceph-mon[23733]: from='client.? 172.21.15.89:0/4236641004' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-09-21T04:01:47.913 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: 2024-09-21T04:01:47.913 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: [global] 2024-09-21T04:01:47.913 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: fsid = 1b9b79fc-77ce-11ef-baf5-efdc52797490 2024-09-21T04:01:47.913 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: mon_osd_allow_pg_remap = true 2024-09-21T04:01:47.913 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: mon_osd_allow_primary_affinity = true 2024-09-21T04:01:47.913 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: mon_warn_on_no_sortbitwise = false 2024-09-21T04:01:47.913 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: osd_crush_chooseleaf_type = 0 2024-09-21T04:01:47.913 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: 2024-09-21T04:01:47.913 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: [mgr] 2024-09-21T04:01:47.913 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: mgr/telemetry/nag = false 2024-09-21T04:01:47.913 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: 2024-09-21T04:01:47.914 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: [osd] 2024-09-21T04:01:47.914 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: osd_map_max_advance = 10 2024-09-21T04:01:47.914 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: osd_sloppy_crc = true 2024-09-21T04:01:48.202 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:47 smithi089 ceph-mon[23733]: mgrmap e4: smithi089.srlyxf(active, since 2s) 2024-09-21T04:01:48.202 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:47 smithi089 ceph-mon[23733]: from='client.? 172.21.15.89:0/3323115261' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2024-09-21T04:01:48.448 INFO:teuthology.orchestra.run.smithi089.stderr:Enabling cephadm module... 2024-09-21T04:01:49.426 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:49 smithi089 ceph-mon[23733]: from='client.? 172.21.15.89:0/3255973980' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "cephadm"}]: dispatch 2024-09-21T04:01:50.264 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:50 smithi089 ceph-mon[23733]: from='client.? 172.21.15.89:0/3255973980' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "cephadm"}]': finished 2024-09-21T04:01:50.264 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:50 smithi089 ceph-mon[23733]: mgrmap e5: smithi089.srlyxf(active, since 4s) 2024-09-21T04:01:50.409 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: { 2024-09-21T04:01:50.409 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "epoch": 5, 2024-09-21T04:01:50.409 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "available": true, 2024-09-21T04:01:50.409 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "active_name": "smithi089.srlyxf", 2024-09-21T04:01:50.409 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "num_standby": 0 2024-09-21T04:01:50.409 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: } 2024-09-21T04:01:51.012 INFO:teuthology.orchestra.run.smithi089.stderr:Waiting for the mgr to restart... 2024-09-21T04:01:51.012 INFO:teuthology.orchestra.run.smithi089.stderr:Waiting for mgr epoch 5... 2024-09-21T04:01:51.264 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:51 smithi089 ceph-mon[23733]: from='client.? 172.21.15.89:0/3419324750' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2024-09-21T04:01:55.514 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:55 smithi089 ceph-mon[23733]: Active manager daemon smithi089.srlyxf restarted 2024-09-21T04:01:55.514 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:55 smithi089 ceph-mon[23733]: Activating manager daemon smithi089.srlyxf 2024-09-21T04:01:55.514 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:55 smithi089 ceph-mon[23733]: osdmap e2: 0 total, 0 up, 0 in 2024-09-21T04:01:56.280 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: { 2024-09-21T04:01:56.280 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "mgrmap_epoch": 7, 2024-09-21T04:01:56.280 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "initialized": true 2024-09-21T04:01:56.280 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: } 2024-09-21T04:01:56.514 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:56 smithi089 ceph-mon[23733]: mgrmap e6: smithi089.srlyxf(active, starting, since 0.0524966s) 2024-09-21T04:01:56.515 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:56 smithi089 ceph-mon[23733]: from='mgr.14118 172.21.15.89:0/2298398671' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "mon metadata", "id": "smithi089"}]: dispatch 2024-09-21T04:01:56.515 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:56 smithi089 ceph-mon[23733]: from='mgr.14118 172.21.15.89:0/2298398671' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "mgr metadata", "who": "smithi089.srlyxf", "id": "smithi089.srlyxf"}]: dispatch 2024-09-21T04:01:56.515 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:56 smithi089 ceph-mon[23733]: from='mgr.14118 172.21.15.89:0/2298398671' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-21T04:01:56.515 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:56 smithi089 ceph-mon[23733]: from='mgr.14118 172.21.15.89:0/2298398671' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-21T04:01:56.515 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:56 smithi089 ceph-mon[23733]: from='mgr.14118 172.21.15.89:0/2298398671' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-21T04:01:56.515 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:56 smithi089 ceph-mon[23733]: Manager daemon smithi089.srlyxf is now available 2024-09-21T04:01:56.515 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:56 smithi089 ceph-mon[23733]: from='mgr.14118 172.21.15.89:0/2298398671' entity='mgr.smithi089.srlyxf' 2024-09-21T04:01:56.515 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:56 smithi089 ceph-mon[23733]: from='mgr.14118 172.21.15.89:0/2298398671' entity='mgr.smithi089.srlyxf' 2024-09-21T04:01:56.515 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:56 smithi089 ceph-mon[23733]: from='mgr.14118 172.21.15.89:0/2298398671' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T04:01:56.515 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:56 smithi089 ceph-mon[23733]: from='mgr.14118 172.21.15.89:0/2298398671' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T04:01:56.515 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:56 smithi089 ceph-mon[23733]: from='mgr.14118 172.21.15.89:0/2298398671' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T04:01:56.515 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:56 smithi089 ceph-mon[23733]: from='mgr.14118 172.21.15.89:0/2298398671' entity='mgr.smithi089.srlyxf' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi089.srlyxf/mirror_snapshot_schedule"}]: dispatch 2024-09-21T04:01:56.515 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:56 smithi089 ceph-mon[23733]: from='mgr.14118 172.21.15.89:0/2298398671' entity='mgr.smithi089.srlyxf' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi089.srlyxf/trash_purge_schedule"}]: dispatch 2024-09-21T04:01:56.919 INFO:teuthology.orchestra.run.smithi089.stderr:mgr epoch 5 is available 2024-09-21T04:01:56.919 INFO:teuthology.orchestra.run.smithi089.stderr:Setting orchestrator backend to cephadm... 2024-09-21T04:01:57.514 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:57 smithi089 ceph-mon[23733]: [21/Sep/2024:04:01:56] ENGINE Bus STARTING 2024-09-21T04:01:57.515 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:57 smithi089 ceph-mon[23733]: [21/Sep/2024:04:01:56] ENGINE Serving on https://172.21.15.89:7150 2024-09-21T04:01:57.515 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:57 smithi089 ceph-mon[23733]: [21/Sep/2024:04:01:56] ENGINE Bus STARTED 2024-09-21T04:01:57.515 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:57 smithi089 ceph-mon[23733]: from='client.14122 -' entity='client.admin' cmd=[{"prefix": "get_command_descriptions"}]: dispatch 2024-09-21T04:01:57.515 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:57 smithi089 ceph-mon[23733]: mgrmap e7: smithi089.srlyxf(active, since 1.05974s) 2024-09-21T04:01:57.515 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:57 smithi089 ceph-mon[23733]: from='mgr.14118 172.21.15.89:0/2298398671' entity='mgr.smithi089.srlyxf' 2024-09-21T04:01:57.515 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:57 smithi089 ceph-mon[23733]: from='client.14122 -' entity='client.admin' cmd=[{"prefix": "mgr_status"}]: dispatch 2024-09-21T04:01:57.515 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:57 smithi089 ceph-mon[23733]: from='mgr.14118 172.21.15.89:0/2298398671' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T04:01:58.669 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:58 smithi089 ceph-mon[23733]: from='client.14130 -' entity='client.admin' cmd=[{"prefix": "orch set backend", "module_name": "cephadm", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T04:01:58.670 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:58 smithi089 ceph-mon[23733]: from='mgr.14118 172.21.15.89:0/2298398671' entity='mgr.smithi089.srlyxf' 2024-09-21T04:01:58.670 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:58 smithi089 ceph-mon[23733]: from='mgr.14118 172.21.15.89:0/2298398671' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T04:01:58.670 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:58 smithi089 ceph-mon[23733]: mgrmap e8: smithi089.srlyxf(active, since 2s) 2024-09-21T04:01:58.670 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: value unchanged 2024-09-21T04:01:59.241 INFO:teuthology.orchestra.run.smithi089.stderr:Generating ssh key... 2024-09-21T04:02:00.014 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:01:59 smithi089 ceph-mon[23733]: from='client.14132 -' entity='client.admin' cmd=[{"prefix": "cephadm set-user", "user": "root", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T04:02:01.147 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:02:01 smithi089 ceph-mon[23733]: from='client.14134 -' entity='client.admin' cmd=[{"prefix": "cephadm generate-key", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T04:02:01.148 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:02:01 smithi089 ceph-mon[23733]: Generating ssh key... 2024-09-21T04:02:01.148 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:02:01 smithi089 ceph-mon[23733]: from='mgr.14118 172.21.15.89:0/2298398671' entity='mgr.smithi089.srlyxf' 2024-09-21T04:02:01.148 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:02:01 smithi089 ceph-mon[23733]: from='mgr.14118 172.21.15.89:0/2298398671' entity='mgr.smithi089.srlyxf' 2024-09-21T04:02:01.148 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCgrbKq9nH+OH7UzhsPruwJShj69/LMZpGDyl0bQuVgCWRcFZ2RTzU6O9tQnXN1NgW3sew2WxpSXA7rJhuRD7zxUqi0XUJAUAk4R5O0wiQTV4AdoF62oycxMoRkYgT1SFu7qDTKJPAshqNvmrn/o8wzWXtTc1shIbgrhEeBrbqVDdKjhxn450rybL2JFLgZ67BQdNmsssK4B7ff2qWUAn5S6I/G3MdKbjF8guSxCL1he7Tj6n4FKQ9Grb+k2uGIO3rgajXAT4POyncVjlZFMvR8BDaqGRZIB41G6bOVdy86YMKqTRoVCDcsexR+xkKk4klSr3uZRHBtrzvuZcteg5DwhxFNOgS3EKIxhMYJDOsVGmrvmsDEofl2qjcjFfJl4L/Fu72UkpQN4W1QYFhBAkoUInHWU9+CcKC3RlewfSUNxGarnGSAiBQCP1cPvnWwbtfms5wN80B+L1V4/gzj2wNHa9TjPZwiVKg7t8TRmPGD7AkpjNYRMmFAv6b1XEVZHN0= ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490 2024-09-21T04:02:01.713 INFO:teuthology.orchestra.run.smithi089.stderr:Wrote public SSH key to /home/ubuntu/cephtest/ceph.pub 2024-09-21T04:02:01.714 INFO:teuthology.orchestra.run.smithi089.stderr:Adding key to root@localhost authorized_keys... 2024-09-21T04:02:01.720 INFO:teuthology.orchestra.run.smithi089.stderr:Adding host smithi089... 2024-09-21T04:02:03.153 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: Added host 'smithi089' with addr '172.21.15.89' 2024-09-21T04:02:03.164 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:02:03 smithi089 ceph-mon[23733]: from='client.14136 -' entity='client.admin' cmd=[{"prefix": "cephadm get-pub-key", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T04:02:03.854 INFO:teuthology.orchestra.run.smithi089.stderr:Deploying mon service with default placement... 2024-09-21T04:02:04.232 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:02:04 smithi089 ceph-mon[23733]: from='client.14138 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi089", "addr": "172.21.15.89", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T04:02:04.232 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:02:04 smithi089 ceph-mon[23733]: Deploying cephadm binary to smithi089 2024-09-21T04:02:04.232 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:02:04 smithi089 ceph-mon[23733]: from='mgr.14118 172.21.15.89:0/2298398671' entity='mgr.smithi089.srlyxf' 2024-09-21T04:02:04.232 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:02:04 smithi089 ceph-mon[23733]: from='mgr.14118 172.21.15.89:0/2298398671' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T04:02:04.529 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: Scheduled mon update... 2024-09-21T04:02:05.126 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:02:05 smithi089 ceph-mon[23733]: Added host smithi089 2024-09-21T04:02:05.127 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:02:05 smithi089 ceph-mon[23733]: from='mgr.14118 172.21.15.89:0/2298398671' entity='mgr.smithi089.srlyxf' 2024-09-21T04:02:06.054 INFO:teuthology.orchestra.run.smithi089.stderr:Deploying mgr service with default placement... 2024-09-21T04:02:06.313 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:02:06 smithi089 ceph-mon[23733]: from='client.14140 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T04:02:06.314 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:02:06 smithi089 ceph-mon[23733]: Saving service mon spec with placement count:5 2024-09-21T04:02:06.314 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:02:06 smithi089 ceph-mon[23733]: from='mgr.14118 172.21.15.89:0/2298398671' entity='mgr.smithi089.srlyxf' 2024-09-21T04:02:06.733 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: Scheduled mgr update... 2024-09-21T04:02:07.483 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:02:07 smithi089 ceph-mon[23733]: from='mgr.14118 172.21.15.89:0/2298398671' entity='mgr.smithi089.srlyxf' 2024-09-21T04:02:07.483 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:02:07 smithi089 ceph-mon[23733]: from='mgr.14118 172.21.15.89:0/2298398671' entity='mgr.smithi089.srlyxf' 2024-09-21T04:02:07.850 INFO:teuthology.orchestra.run.smithi089.stderr:Deploying crash service with default placement... 2024-09-21T04:02:08.394 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:02:08 smithi089 ceph-mon[23733]: from='client.14142 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T04:02:08.394 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:02:08 smithi089 ceph-mon[23733]: Saving service mgr spec with placement count:2 2024-09-21T04:02:08.394 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:02:08 smithi089 ceph-mon[23733]: from='mgr.14118 172.21.15.89:0/2298398671' entity='mgr.smithi089.srlyxf' 2024-09-21T04:02:08.525 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: Scheduled crash update... 2024-09-21T04:02:09.100 INFO:teuthology.orchestra.run.smithi089.stderr:Deploying prometheus service with default placement... 2024-09-21T04:02:09.679 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:02:09 smithi089 ceph-mon[23733]: from='mgr.14118 172.21.15.89:0/2298398671' entity='mgr.smithi089.srlyxf' 2024-09-21T04:02:09.679 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:02:09 smithi089 ceph-mon[23733]: from='client.14144 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "crash", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T04:02:09.679 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:02:09 smithi089 ceph-mon[23733]: Saving service crash spec with placement * 2024-09-21T04:02:09.679 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:02:09 smithi089 ceph-mon[23733]: from='mgr.14118 172.21.15.89:0/2298398671' entity='mgr.smithi089.srlyxf' 2024-09-21T04:02:09.745 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: Scheduled prometheus update... 2024-09-21T04:02:10.292 INFO:teuthology.orchestra.run.smithi089.stderr:Deploying grafana service with default placement... 2024-09-21T04:02:10.887 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:02:10 smithi089 ceph-mon[23733]: from='client.14146 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T04:02:10.887 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:02:10 smithi089 ceph-mon[23733]: Saving service prometheus spec with placement count:1 2024-09-21T04:02:10.887 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:02:10 smithi089 ceph-mon[23733]: from='mgr.14118 172.21.15.89:0/2298398671' entity='mgr.smithi089.srlyxf' 2024-09-21T04:02:10.888 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: Scheduled grafana update... 2024-09-21T04:02:11.676 INFO:teuthology.orchestra.run.smithi089.stderr:Deploying node-exporter service with default placement... 2024-09-21T04:02:12.008 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:02:11 smithi089 ceph-mon[23733]: from='client.14148 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T04:02:12.008 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:02:11 smithi089 ceph-mon[23733]: Saving service grafana spec with placement count:1 2024-09-21T04:02:12.008 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:02:11 smithi089 ceph-mon[23733]: from='mgr.14118 172.21.15.89:0/2298398671' entity='mgr.smithi089.srlyxf' 2024-09-21T04:02:12.612 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: Scheduled node-exporter update... 2024-09-21T04:02:13.202 INFO:teuthology.orchestra.run.smithi089.stderr:Deploying alertmanager service with default placement... 2024-09-21T04:02:13.831 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:02:13 smithi089 ceph-mon[23733]: from='client.14150 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "node-exporter", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T04:02:13.831 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:02:13 smithi089 ceph-mon[23733]: Saving service node-exporter spec with placement * 2024-09-21T04:02:13.831 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:02:13 smithi089 ceph-mon[23733]: from='mgr.14118 172.21.15.89:0/2298398671' entity='mgr.smithi089.srlyxf' 2024-09-21T04:02:13.831 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: Scheduled alertmanager update... 2024-09-21T04:02:14.993 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:02:14 smithi089 ceph-mon[23733]: from='client.14152 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T04:02:14.993 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:02:14 smithi089 ceph-mon[23733]: Saving service alertmanager spec with placement count:1 2024-09-21T04:02:14.993 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:02:14 smithi089 ceph-mon[23733]: from='mgr.14118 172.21.15.89:0/2298398671' entity='mgr.smithi089.srlyxf' 2024-09-21T04:02:16.227 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:02:15 smithi089 ceph-mon[23733]: from='client.? 172.21.15.89:0/1409651560' entity='client.admin' 2024-09-21T04:02:16.787 INFO:teuthology.orchestra.run.smithi089.stderr:Enabling the dashboard module... 2024-09-21T04:02:17.050 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:02:16 smithi089 ceph-mon[23733]: pgmap v3: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T04:02:17.050 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:02:16 smithi089 ceph-mon[23733]: from='client.? 172.21.15.89:0/1413692001' entity='client.admin' 2024-09-21T04:02:18.307 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:02:18 smithi089 ceph-mon[23733]: from='client.? 172.21.15.89:0/3767697570' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "dashboard"}]: dispatch 2024-09-21T04:02:19.154 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:02:19 smithi089 ceph-mon[23733]: from='client.? 172.21.15.89:0/3767697570' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "dashboard"}]': finished 2024-09-21T04:02:19.154 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:02:19 smithi089 ceph-mon[23733]: mgrmap e9: smithi089.srlyxf(active, since 22s) 2024-09-21T04:02:19.399 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: { 2024-09-21T04:02:19.399 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "epoch": 9, 2024-09-21T04:02:19.400 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "available": true, 2024-09-21T04:02:19.400 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "active_name": "smithi089.srlyxf", 2024-09-21T04:02:19.400 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "num_standby": 0 2024-09-21T04:02:19.400 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: } 2024-09-21T04:02:19.960 INFO:teuthology.orchestra.run.smithi089.stderr:Waiting for the mgr to restart... 2024-09-21T04:02:19.960 INFO:teuthology.orchestra.run.smithi089.stderr:Waiting for mgr epoch 9... 2024-09-21T04:02:20.352 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:02:20 smithi089 ceph-mon[23733]: from='client.? 172.21.15.89:0/1803815938' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2024-09-21T04:02:24.379 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:02:24 smithi089 ceph-mon[23733]: Active manager daemon smithi089.srlyxf restarted 2024-09-21T04:02:24.380 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:02:24 smithi089 ceph-mon[23733]: Activating manager daemon smithi089.srlyxf 2024-09-21T04:02:24.380 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:02:24 smithi089 ceph-mon[23733]: osdmap e3: 0 total, 0 up, 0 in 2024-09-21T04:02:25.316 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: { 2024-09-21T04:02:25.316 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "mgrmap_epoch": 11, 2024-09-21T04:02:25.316 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "initialized": true 2024-09-21T04:02:25.316 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: } 2024-09-21T04:02:25.743 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:02:25 smithi089 ceph-mon[23733]: mgrmap e10: smithi089.srlyxf(active, starting, since 0.054925s) 2024-09-21T04:02:25.743 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:02:25 smithi089 ceph-mon[23733]: from='mgr.14160 172.21.15.89:0/4274291427' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "mon metadata", "id": "smithi089"}]: dispatch 2024-09-21T04:02:25.743 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:02:25 smithi089 ceph-mon[23733]: from='mgr.14160 172.21.15.89:0/4274291427' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "mgr metadata", "who": "smithi089.srlyxf", "id": "smithi089.srlyxf"}]: dispatch 2024-09-21T04:02:25.743 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:02:25 smithi089 ceph-mon[23733]: from='mgr.14160 172.21.15.89:0/4274291427' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-21T04:02:25.743 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:02:25 smithi089 ceph-mon[23733]: from='mgr.14160 172.21.15.89:0/4274291427' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-21T04:02:25.743 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:02:25 smithi089 ceph-mon[23733]: from='mgr.14160 172.21.15.89:0/4274291427' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-21T04:02:25.743 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:02:25 smithi089 ceph-mon[23733]: Manager daemon smithi089.srlyxf is now available 2024-09-21T04:02:25.743 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:02:25 smithi089 ceph-mon[23733]: from='mgr.14160 172.21.15.89:0/4274291427' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T04:02:25.743 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:02:25 smithi089 ceph-mon[23733]: from='mgr.14160 172.21.15.89:0/4274291427' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T04:02:25.744 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:02:25 smithi089 ceph-mon[23733]: from='mgr.14160 172.21.15.89:0/4274291427' entity='mgr.smithi089.srlyxf' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi089.srlyxf/mirror_snapshot_schedule"}]: dispatch 2024-09-21T04:02:25.744 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:02:25 smithi089 ceph-mon[23733]: from='mgr.14160 172.21.15.89:0/4274291427' entity='mgr.smithi089.srlyxf' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi089.srlyxf/trash_purge_schedule"}]: dispatch 2024-09-21T04:02:25.744 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:02:25 smithi089 ceph-mon[23733]: [21/Sep/2024:04:02:24] ENGINE Bus STARTING 2024-09-21T04:02:25.744 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:02:25 smithi089 ceph-mon[23733]: [21/Sep/2024:04:02:24] ENGINE Serving on https://172.21.15.89:7150 2024-09-21T04:02:25.744 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:02:25 smithi089 ceph-mon[23733]: [21/Sep/2024:04:02:24] ENGINE Bus STARTED 2024-09-21T04:02:25.744 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:02:25 smithi089 ceph-mon[23733]: from='mgr.14160 172.21.15.89:0/4274291427' entity='mgr.smithi089.srlyxf' 2024-09-21T04:02:26.020 INFO:teuthology.orchestra.run.smithi089.stderr:mgr epoch 9 is available 2024-09-21T04:02:26.020 INFO:teuthology.orchestra.run.smithi089.stderr:Generating a dashboard self-signed certificate... 2024-09-21T04:02:26.563 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:02:26 smithi089 ceph-mon[23733]: from='client.14164 -' entity='client.admin' cmd=[{"prefix": "get_command_descriptions"}]: dispatch 2024-09-21T04:02:26.563 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:02:26 smithi089 ceph-mon[23733]: mgrmap e11: smithi089.srlyxf(active, since 1.06548s) 2024-09-21T04:02:26.563 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:02:26 smithi089 ceph-mon[23733]: from='client.14164 -' entity='client.admin' cmd=[{"prefix": "mgr_status"}]: dispatch 2024-09-21T04:02:26.781 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: Self-signed certificate created 2024-09-21T04:02:27.360 INFO:teuthology.orchestra.run.smithi089.stderr:Creating initial admin user... 2024-09-21T04:02:27.481 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:02:27 smithi089 ceph-mon[23733]: mgrmap e12: smithi089.srlyxf(active, since 2s) 2024-09-21T04:02:27.481 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:02:27 smithi089 ceph-mon[23733]: from='mgr.14160 172.21.15.89:0/4274291427' entity='mgr.smithi089.srlyxf' 2024-09-21T04:02:27.481 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:02:27 smithi089 ceph-mon[23733]: from='client.14172 -' entity='client.admin' cmd=[{"prefix": "dashboard create-self-signed-cert", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T04:02:27.482 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:02:27 smithi089 ceph-mon[23733]: from='mgr.14160 172.21.15.89:0/4274291427' entity='mgr.smithi089.srlyxf' 2024-09-21T04:02:27.482 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:02:27 smithi089 ceph-mon[23733]: from='mgr.14160 172.21.15.89:0/4274291427' entity='mgr.smithi089.srlyxf' 2024-09-21T04:02:28.254 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: {"username": "admin", "password": "$2b$12$DbUcbbrdDe4esJ9.ame8JOQ9qhh1hAdx44Ask87Lo9G9XHXYV.mim", "roles": ["administrator"], "name": null, "email": null, "lastUpdate": 1726891348, "enabled": true, "pwdExpirationDate": null, "pwdUpdateRequired": true} 2024-09-21T04:02:28.828 INFO:teuthology.orchestra.run.smithi089.stderr:Fetching dashboard port number... 2024-09-21T04:02:29.083 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:02:28 smithi089 ceph-mon[23733]: from='mgr.14160 172.21.15.89:0/4274291427' entity='mgr.smithi089.srlyxf' 2024-09-21T04:02:29.084 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:02:28 smithi089 ceph-mon[23733]: from='client.14174 -' entity='client.admin' cmd=[{"prefix": "dashboard ac-user-create", "username": "admin", "rolename": "administrator", "force_password": true, "pwd_update_required": true, "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T04:02:29.084 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:02:28 smithi089 ceph-mon[23733]: from='mgr.14160 172.21.15.89:0/4274291427' entity='mgr.smithi089.srlyxf' 2024-09-21T04:02:29.619 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: 8443 2024-09-21T04:02:30.177 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:02:29 smithi089 ceph-mon[23733]: from='client.? 172.21.15.89:0/1673316355' entity='client.admin' cmd=[{"prefix": "config get", "who": "mgr", "key": "mgr/dashboard/ssl_server_port"}]: dispatch 2024-09-21T04:02:30.834 INFO:teuthology.orchestra.run.smithi089.stderr:systemctl: disabled 2024-09-21T04:02:30.843 INFO:teuthology.orchestra.run.smithi089.stderr:systemctl: inactive 2024-09-21T04:02:30.843 INFO:teuthology.orchestra.run.smithi089.stderr:firewalld.service is not enabled 2024-09-21T04:02:30.844 INFO:teuthology.orchestra.run.smithi089.stderr:Not possible to open ports <[8443]>. firewalld.service is not available 2024-09-21T04:02:30.846 INFO:teuthology.orchestra.run.smithi089.stderr:Ceph Dashboard is now available at: 2024-09-21T04:02:30.846 INFO:teuthology.orchestra.run.smithi089.stderr: 2024-09-21T04:02:30.846 INFO:teuthology.orchestra.run.smithi089.stderr: URL: https://smithi089.front.sepia.ceph.com:8443/ 2024-09-21T04:02:30.846 INFO:teuthology.orchestra.run.smithi089.stderr: User: admin 2024-09-21T04:02:30.847 INFO:teuthology.orchestra.run.smithi089.stderr: Password: sm33dxdufi 2024-09-21T04:02:30.847 INFO:teuthology.orchestra.run.smithi089.stderr: 2024-09-21T04:02:30.847 INFO:teuthology.orchestra.run.smithi089.stderr:Enabling autotune for osd_memory_target 2024-09-21T04:02:32.087 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:02:31 smithi089 ceph-mon[23733]: mgrmap e13: smithi089.srlyxf(active, since 6s) 2024-09-21T04:02:32.737 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: set mgr/dashboard/cluster/status 2024-09-21T04:02:33.338 INFO:teuthology.orchestra.run.smithi089.stderr:You can access the Ceph CLI with: 2024-09-21T04:02:33.338 INFO:teuthology.orchestra.run.smithi089.stderr: 2024-09-21T04:02:33.338 INFO:teuthology.orchestra.run.smithi089.stderr: sudo /home/ubuntu/cephtest/cephadm shell --fsid 1b9b79fc-77ce-11ef-baf5-efdc52797490 -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring 2024-09-21T04:02:33.338 INFO:teuthology.orchestra.run.smithi089.stderr: 2024-09-21T04:02:33.338 INFO:teuthology.orchestra.run.smithi089.stderr:Please consider enabling telemetry to help improve Ceph: 2024-09-21T04:02:33.339 INFO:teuthology.orchestra.run.smithi089.stderr: 2024-09-21T04:02:33.339 INFO:teuthology.orchestra.run.smithi089.stderr: ceph telemetry on 2024-09-21T04:02:33.339 INFO:teuthology.orchestra.run.smithi089.stderr: 2024-09-21T04:02:33.339 INFO:teuthology.orchestra.run.smithi089.stderr:For more information see: 2024-09-21T04:02:33.339 INFO:teuthology.orchestra.run.smithi089.stderr: 2024-09-21T04:02:33.339 INFO:teuthology.orchestra.run.smithi089.stderr: https://docs.ceph.com/docs/master/mgr/telemetry/ 2024-09-21T04:02:33.339 INFO:teuthology.orchestra.run.smithi089.stderr: 2024-09-21T04:02:33.339 INFO:teuthology.orchestra.run.smithi089.stderr:Bootstrap complete. 2024-09-21T04:02:33.372 INFO:tasks.cephadm:Fetching config... 2024-09-21T04:02:33.372 DEBUG:teuthology.orchestra.run.smithi089:> set -ex 2024-09-21T04:02:33.372 DEBUG:teuthology.orchestra.run.smithi089:> dd if=/etc/ceph/ceph.conf of=/dev/stdout 2024-09-21T04:02:33.388 INFO:tasks.cephadm:Fetching client.admin keyring... 2024-09-21T04:02:33.388 DEBUG:teuthology.orchestra.run.smithi089:> set -ex 2024-09-21T04:02:33.388 DEBUG:teuthology.orchestra.run.smithi089:> dd if=/etc/ceph/ceph.client.admin.keyring of=/dev/stdout 2024-09-21T04:02:33.443 INFO:tasks.cephadm:Fetching mon keyring... 2024-09-21T04:02:33.444 DEBUG:teuthology.orchestra.run.smithi089:> set -ex 2024-09-21T04:02:33.444 DEBUG:teuthology.orchestra.run.smithi089:> sudo dd if=/var/lib/ceph/1b9b79fc-77ce-11ef-baf5-efdc52797490/mon.smithi089/keyring of=/dev/stdout 2024-09-21T04:02:33.509 INFO:tasks.cephadm:Fetching pub ssh key... 2024-09-21T04:02:33.509 DEBUG:teuthology.orchestra.run.smithi089:> set -ex 2024-09-21T04:02:33.509 DEBUG:teuthology.orchestra.run.smithi089:> dd if=/home/ubuntu/cephtest/ceph.pub of=/dev/stdout 2024-09-21T04:02:33.564 INFO:tasks.cephadm:Installing pub ssh key for root users... 2024-09-21T04:02:33.564 DEBUG:teuthology.orchestra.run.smithi089:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCgrbKq9nH+OH7UzhsPruwJShj69/LMZpGDyl0bQuVgCWRcFZ2RTzU6O9tQnXN1NgW3sew2WxpSXA7rJhuRD7zxUqi0XUJAUAk4R5O0wiQTV4AdoF62oycxMoRkYgT1SFu7qDTKJPAshqNvmrn/o8wzWXtTc1shIbgrhEeBrbqVDdKjhxn450rybL2JFLgZ67BQdNmsssK4B7ff2qWUAn5S6I/G3MdKbjF8guSxCL1he7Tj6n4FKQ9Grb+k2uGIO3rgajXAT4POyncVjlZFMvR8BDaqGRZIB41G6bOVdy86YMKqTRoVCDcsexR+xkKk4klSr3uZRHBtrzvuZcteg5DwhxFNOgS3EKIxhMYJDOsVGmrvmsDEofl2qjcjFfJl4L/Fu72UkpQN4W1QYFhBAkoUInHWU9+CcKC3RlewfSUNxGarnGSAiBQCP1cPvnWwbtfms5wN80B+L1V4/gzj2wNHa9TjPZwiVKg7t8TRmPGD7AkpjNYRMmFAv6b1XEVZHN0= ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2024-09-21T04:02:33.641 INFO:teuthology.orchestra.run.smithi089.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCgrbKq9nH+OH7UzhsPruwJShj69/LMZpGDyl0bQuVgCWRcFZ2RTzU6O9tQnXN1NgW3sew2WxpSXA7rJhuRD7zxUqi0XUJAUAk4R5O0wiQTV4AdoF62oycxMoRkYgT1SFu7qDTKJPAshqNvmrn/o8wzWXtTc1shIbgrhEeBrbqVDdKjhxn450rybL2JFLgZ67BQdNmsssK4B7ff2qWUAn5S6I/G3MdKbjF8guSxCL1he7Tj6n4FKQ9Grb+k2uGIO3rgajXAT4POyncVjlZFMvR8BDaqGRZIB41G6bOVdy86YMKqTRoVCDcsexR+xkKk4klSr3uZRHBtrzvuZcteg5DwhxFNOgS3EKIxhMYJDOsVGmrvmsDEofl2qjcjFfJl4L/Fu72UkpQN4W1QYFhBAkoUInHWU9+CcKC3RlewfSUNxGarnGSAiBQCP1cPvnWwbtfms5wN80B+L1V4/gzj2wNHa9TjPZwiVKg7t8TRmPGD7AkpjNYRMmFAv6b1XEVZHN0= ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490 2024-09-21T04:02:33.653 DEBUG:teuthology.orchestra.run.smithi195:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCgrbKq9nH+OH7UzhsPruwJShj69/LMZpGDyl0bQuVgCWRcFZ2RTzU6O9tQnXN1NgW3sew2WxpSXA7rJhuRD7zxUqi0XUJAUAk4R5O0wiQTV4AdoF62oycxMoRkYgT1SFu7qDTKJPAshqNvmrn/o8wzWXtTc1shIbgrhEeBrbqVDdKjhxn450rybL2JFLgZ67BQdNmsssK4B7ff2qWUAn5S6I/G3MdKbjF8guSxCL1he7Tj6n4FKQ9Grb+k2uGIO3rgajXAT4POyncVjlZFMvR8BDaqGRZIB41G6bOVdy86YMKqTRoVCDcsexR+xkKk4klSr3uZRHBtrzvuZcteg5DwhxFNOgS3EKIxhMYJDOsVGmrvmsDEofl2qjcjFfJl4L/Fu72UkpQN4W1QYFhBAkoUInHWU9+CcKC3RlewfSUNxGarnGSAiBQCP1cPvnWwbtfms5wN80B+L1V4/gzj2wNHa9TjPZwiVKg7t8TRmPGD7AkpjNYRMmFAv6b1XEVZHN0= ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2024-09-21T04:02:33.705 INFO:teuthology.orchestra.run.smithi195.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCgrbKq9nH+OH7UzhsPruwJShj69/LMZpGDyl0bQuVgCWRcFZ2RTzU6O9tQnXN1NgW3sew2WxpSXA7rJhuRD7zxUqi0XUJAUAk4R5O0wiQTV4AdoF62oycxMoRkYgT1SFu7qDTKJPAshqNvmrn/o8wzWXtTc1shIbgrhEeBrbqVDdKjhxn450rybL2JFLgZ67BQdNmsssK4B7ff2qWUAn5S6I/G3MdKbjF8guSxCL1he7Tj6n4FKQ9Grb+k2uGIO3rgajXAT4POyncVjlZFMvR8BDaqGRZIB41G6bOVdy86YMKqTRoVCDcsexR+xkKk4klSr3uZRHBtrzvuZcteg5DwhxFNOgS3EKIxhMYJDOsVGmrvmsDEofl2qjcjFfJl4L/Fu72UkpQN4W1QYFhBAkoUInHWU9+CcKC3RlewfSUNxGarnGSAiBQCP1cPvnWwbtfms5wN80B+L1V4/gzj2wNHa9TjPZwiVKg7t8TRmPGD7AkpjNYRMmFAv6b1XEVZHN0= ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490 2024-09-21T04:02:33.718 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 1b9b79fc-77ce-11ef-baf5-efdc52797490 -- ceph config set mgr mgr/cephadm/allow_ptrace true 2024-09-21T04:02:34.014 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:02:33 smithi089 ceph-mon[23733]: from='client.? 172.21.15.89:0/2743807179' entity='client.admin' 2024-09-21T04:02:37.478 INFO:tasks.cephadm:Distributing conf and client.admin keyring to all hosts + 0755 2024-09-21T04:02:37.478 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 1b9b79fc-77ce-11ef-baf5-efdc52797490 -- ceph orch client-keyring set client.admin '*' --mode 0755 2024-09-21T04:02:37.764 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:02:37 smithi089 ceph-mon[23733]: from='client.? 172.21.15.89:0/3079331656' entity='client.admin' 2024-09-21T04:02:39.096 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:02:38 smithi089 ceph-mon[23733]: from='mgr.14160 172.21.15.89:0/4274291427' entity='mgr.smithi089.srlyxf' 2024-09-21T04:02:39.096 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:02:38 smithi089 ceph-mon[23733]: from='mgr.14160 172.21.15.89:0/4274291427' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "config rm", "who": "osd/host:smithi089", "name": "osd_memory_target"}]: dispatch 2024-09-21T04:02:39.096 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:02:38 smithi089 ceph-mon[23733]: from='mgr.14160 172.21.15.89:0/4274291427' entity='mgr.smithi089.srlyxf' 2024-09-21T04:02:39.096 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:02:38 smithi089 ceph-mon[23733]: Deploying daemon alertmanager.smithi089 on smithi089 2024-09-21T04:02:40.184 INFO:tasks.cephadm:Writing (initial) conf and keyring to smithi195 2024-09-21T04:02:40.184 DEBUG:teuthology.orchestra.run.smithi195:> set -ex 2024-09-21T04:02:40.185 DEBUG:teuthology.orchestra.run.smithi195:> dd of=/etc/ceph/ceph.conf 2024-09-21T04:02:40.203 DEBUG:teuthology.orchestra.run.smithi195:> set -ex 2024-09-21T04:02:40.204 DEBUG:teuthology.orchestra.run.smithi195:> dd of=/etc/ceph/ceph.client.admin.keyring 2024-09-21T04:02:40.261 INFO:tasks.cephadm:Adding host smithi195 to orchestrator... 2024-09-21T04:02:40.261 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 1b9b79fc-77ce-11ef-baf5-efdc52797490 -- ceph orch host add smithi195 2024-09-21T04:02:40.558 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:02:40 smithi089 ceph-mon[23733]: from='client.14184 -' entity='client.admin' cmd=[{"prefix": "orch client-keyring set", "entity": "client.admin", "placement": "*", "mode": "0755", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T04:02:40.559 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:02:40 smithi089 ceph-mon[23733]: from='mgr.14160 172.21.15.89:0/4274291427' entity='mgr.smithi089.srlyxf' 2024-09-21T04:02:44.438 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:02:44 smithi089 ceph-mon[23733]: from='client.14186 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi195", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T04:02:44.589 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:02:44 smithi089 ceph-mon[23733]: Deploying cephadm binary to smithi195 2024-09-21T04:02:44.590 INFO:teuthology.orchestra.run.smithi089.stdout:Added host 'smithi195' with addr '172.21.15.195' 2024-09-21T04:02:45.670 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:02:45 smithi089 ceph-mon[23733]: pgmap v3: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T04:02:45.671 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:02:45 smithi089 ceph-mon[23733]: from='mgr.14160 172.21.15.89:0/4274291427' entity='mgr.smithi089.srlyxf' 2024-09-21T04:02:45.671 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:02:45 smithi089 ceph-mon[23733]: Added host smithi195 2024-09-21T04:02:45.693 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 1b9b79fc-77ce-11ef-baf5-efdc52797490 -- ceph orch host ls --format=json 2024-09-21T04:02:48.093 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:02:48 smithi089 ceph-mon[23733]: pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T04:02:48.384 INFO:teuthology.orchestra.run.smithi089.stdout: 2024-09-21T04:02:48.384 INFO:teuthology.orchestra.run.smithi089.stdout:[{"addr": "172.21.15.89", "hostname": "smithi089", "labels": [], "status": ""}, {"addr": "172.21.15.195", "hostname": "smithi195", "labels": [], "status": ""}] 2024-09-21T04:02:49.491 INFO:tasks.cephadm:Setting crush tunables to default 2024-09-21T04:02:49.491 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 1b9b79fc-77ce-11ef-baf5-efdc52797490 -- ceph osd crush tunables default 2024-09-21T04:02:50.422 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:02:50 smithi089 ceph-mon[23733]: pgmap v5: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T04:02:50.422 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:02:50 smithi089 ceph-mon[23733]: from='client.14188 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T04:02:50.422 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:02:50 smithi089 ceph-mon[23733]: from='mgr.14160 172.21.15.89:0/4274291427' entity='mgr.smithi089.srlyxf' 2024-09-21T04:02:50.422 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:02:50 smithi089 ceph-mon[23733]: from='mgr.14160 172.21.15.89:0/4274291427' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi089", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-09-21T04:02:50.422 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:02:50 smithi089 ceph-mon[23733]: from='mgr.14160 172.21.15.89:0/4274291427' entity='mgr.smithi089.srlyxf' cmd='[{"prefix": "auth get-or-create", "entity": "client.crash.smithi089", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]': finished 2024-09-21T04:02:50.422 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:02:50 smithi089 ceph-mon[23733]: from='mgr.14160 172.21.15.89:0/4274291427' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T04:02:50.422 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:02:50 smithi089 ceph-mon[23733]: Deploying daemon crash.smithi089 on smithi089 2024-09-21T04:02:50.423 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:02:50 smithi089 ceph-mon[23733]: from='mgr.14160 172.21.15.89:0/4274291427' entity='mgr.smithi089.srlyxf' 2024-09-21T04:02:52.359 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:02:52 smithi089 ceph-mon[23733]: pgmap v6: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T04:02:53.071 INFO:teuthology.orchestra.run.smithi089.stderr:adjusted tunables profile to default 2024-09-21T04:02:53.391 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:02:53 smithi089 ceph-mon[23733]: from='client.? 172.21.15.89:0/1999633870' entity='client.admin' cmd=[{"prefix": "osd crush tunables", "profile": "default"}]: dispatch 2024-09-21T04:02:53.391 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:02:53 smithi089 ceph-mon[23733]: from='mgr.14160 172.21.15.89:0/4274291427' entity='mgr.smithi089.srlyxf' 2024-09-21T04:02:53.391 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:02:53 smithi089 ceph-mon[23733]: from='mgr.14160 172.21.15.89:0/4274291427' entity='mgr.smithi089.srlyxf' 2024-09-21T04:02:53.391 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:02:53 smithi089 ceph-mon[23733]: from='mgr.14160 172.21.15.89:0/4274291427' entity='mgr.smithi089.srlyxf' 2024-09-21T04:02:53.391 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:02:53 smithi089 ceph-mon[23733]: from='mgr.14160 172.21.15.89:0/4274291427' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-21T04:02:53.392 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:02:53 smithi089 ceph-mon[23733]: from='mgr.14160 172.21.15.89:0/4274291427' entity='mgr.smithi089.srlyxf' 2024-09-21T04:02:53.717 INFO:tasks.cephadm:Adding mon.smithi089 on smithi089 2024-09-21T04:02:53.717 INFO:tasks.cephadm:Adding mon.smithi195 on smithi195 2024-09-21T04:02:53.717 DEBUG:teuthology.orchestra.run.smithi195:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 1b9b79fc-77ce-11ef-baf5-efdc52797490 -- ceph orch apply mon '2;smithi089:172.21.15.89=smithi089;smithi195:172.21.15.195=smithi195' 2024-09-21T04:02:54.514 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:02:54 smithi089 ceph-mon[23733]: pgmap v7: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T04:02:54.514 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:02:54 smithi089 ceph-mon[23733]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-21T04:02:54.514 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:02:54 smithi089 ceph-mon[23733]: Deploying daemon grafana.smithi089 on smithi089 2024-09-21T04:02:54.514 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:02:54 smithi089 ceph-mon[23733]: from='client.? 172.21.15.89:0/1999633870' entity='client.admin' cmd='[{"prefix": "osd crush tunables", "profile": "default"}]': finished 2024-09-21T04:02:54.515 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:02:54 smithi089 ceph-mon[23733]: osdmap e4: 0 total, 0 up, 0 in 2024-09-21T04:02:55.544 INFO:teuthology.orchestra.run.smithi195.stdout:Scheduled mon update... 2024-09-21T04:02:55.764 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:02:55 smithi089 ceph-mon[23733]: pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T04:02:55.764 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:02:55 smithi089 ceph-mon[23733]: from='mgr.14160 172.21.15.89:0/4274291427' entity='mgr.smithi089.srlyxf' 2024-09-21T04:02:56.139 DEBUG:teuthology.orchestra.run.smithi195:mon.smithi195> sudo journalctl -f -n 0 -u ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490@mon.smithi195.service 2024-09-21T04:02:56.142 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-21T04:02:56.142 DEBUG:teuthology.orchestra.run.smithi195:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 1b9b79fc-77ce-11ef-baf5-efdc52797490 -- ceph mon dump -f json 2024-09-21T04:02:57.014 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:02:56 smithi089 ceph-mon[23733]: from='client.14192 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "placement": "2;smithi089:172.21.15.89=smithi089;smithi195:172.21.15.195=smithi195", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T04:02:57.014 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:02:56 smithi089 ceph-mon[23733]: Saving service mon spec with placement smithi089:172.21.15.89=smithi089;smithi195:172.21.15.195=smithi195;count:2 2024-09-21T04:02:57.014 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:02:56 smithi089 ceph-mon[23733]: from='mgr.14160 172.21.15.89:0/4274291427' entity='mgr.smithi089.srlyxf' 2024-09-21T04:02:57.895 INFO:teuthology.orchestra.run.smithi195.stdout: 2024-09-21T04:02:57.895 INFO:teuthology.orchestra.run.smithi195.stdout:{"epoch":1,"fsid":"1b9b79fc-77ce-11ef-baf5-efdc52797490","modified":"2024-09-21T04:01:28.028960Z","created":"2024-09-21T04:01:28.028960Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi089","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:3300","nonce":0},{"type":"v1","addr":"172.21.15.89:6789","nonce":0}]},"addr":"172.21.15.89:6789/0","public_addr":"172.21.15.89:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-21T04:02:57.897 INFO:teuthology.orchestra.run.smithi195.stderr:dumped monmap epoch 1 2024-09-21T04:02:58.514 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:02:58 smithi089 ceph-mon[23733]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T04:02:58.514 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:02:58 smithi089 ceph-mon[23733]: from='client.? 172.21.15.195:0/3757172449' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-21T04:02:59.409 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-21T04:02:59.410 DEBUG:teuthology.orchestra.run.smithi195:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 1b9b79fc-77ce-11ef-baf5-efdc52797490 -- ceph mon dump -f json 2024-09-21T04:03:00.398 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:03:00 smithi089 ceph-mon[23733]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T04:03:01.249 INFO:teuthology.orchestra.run.smithi195.stdout: 2024-09-21T04:03:01.249 INFO:teuthology.orchestra.run.smithi195.stdout:{"epoch":1,"fsid":"1b9b79fc-77ce-11ef-baf5-efdc52797490","modified":"2024-09-21T04:01:28.028960Z","created":"2024-09-21T04:01:28.028960Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi089","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:3300","nonce":0},{"type":"v1","addr":"172.21.15.89:6789","nonce":0}]},"addr":"172.21.15.89:6789/0","public_addr":"172.21.15.89:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-21T04:03:01.251 INFO:teuthology.orchestra.run.smithi195.stderr:dumped monmap epoch 1 2024-09-21T04:03:02.469 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:03:02 smithi089 ceph-mon[23733]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T04:03:02.469 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:03:02 smithi089 ceph-mon[23733]: from='client.? 172.21.15.195:0/77623644' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-21T04:03:02.811 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-21T04:03:02.811 DEBUG:teuthology.orchestra.run.smithi195:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 1b9b79fc-77ce-11ef-baf5-efdc52797490 -- ceph mon dump -f json 2024-09-21T04:03:04.264 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:03:04 smithi089 ceph-mon[23733]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T04:03:04.265 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:03:04 smithi089 ceph-mon[23733]: from='mgr.14160 172.21.15.89:0/4274291427' entity='mgr.smithi089.srlyxf' 2024-09-21T04:03:04.265 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:03:04 smithi089 ceph-mon[23733]: Deploying daemon node-exporter.smithi089 on smithi089 2024-09-21T04:03:04.568 INFO:teuthology.orchestra.run.smithi195.stdout: 2024-09-21T04:03:04.569 INFO:teuthology.orchestra.run.smithi195.stdout:{"epoch":1,"fsid":"1b9b79fc-77ce-11ef-baf5-efdc52797490","modified":"2024-09-21T04:01:28.028960Z","created":"2024-09-21T04:01:28.028960Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi089","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:3300","nonce":0},{"type":"v1","addr":"172.21.15.89:6789","nonce":0}]},"addr":"172.21.15.89:6789/0","public_addr":"172.21.15.89:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-21T04:03:04.571 INFO:teuthology.orchestra.run.smithi195.stderr:dumped monmap epoch 1 2024-09-21T04:03:05.514 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:03:05 smithi089 ceph-mon[23733]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T04:03:05.514 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:03:05 smithi089 ceph-mon[23733]: from='mgr.14160 172.21.15.89:0/4274291427' entity='mgr.smithi089.srlyxf' 2024-09-21T04:03:05.514 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:03:05 smithi089 ceph-mon[23733]: from='client.? 172.21.15.195:0/3902036469' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-21T04:03:06.203 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-21T04:03:06.203 DEBUG:teuthology.orchestra.run.smithi195:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 1b9b79fc-77ce-11ef-baf5-efdc52797490 -- ceph mon dump -f json 2024-09-21T04:03:07.264 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:03:06 smithi089 ceph-mon[23733]: from='mgr.14160 172.21.15.89:0/4274291427' entity='mgr.smithi089.srlyxf' 2024-09-21T04:03:07.264 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:03:06 smithi089 ceph-mon[23733]: from='mgr.14160 172.21.15.89:0/4274291427' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2024-09-21T04:03:08.055 INFO:teuthology.orchestra.run.smithi195.stdout: 2024-09-21T04:03:08.055 INFO:teuthology.orchestra.run.smithi195.stdout:{"epoch":1,"fsid":"1b9b79fc-77ce-11ef-baf5-efdc52797490","modified":"2024-09-21T04:01:28.028960Z","created":"2024-09-21T04:01:28.028960Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi089","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:3300","nonce":0},{"type":"v1","addr":"172.21.15.89:6789","nonce":0}]},"addr":"172.21.15.89:6789/0","public_addr":"172.21.15.89:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-21T04:03:08.056 INFO:teuthology.orchestra.run.smithi195.stderr:dumped monmap epoch 1 2024-09-21T04:03:08.133 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:03:07 smithi089 ceph-mon[23733]: from='mgr.14160 172.21.15.89:0/4274291427' entity='mgr.smithi089.srlyxf' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2024-09-21T04:03:08.134 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:03:07 smithi089 ceph-mon[23733]: mgrmap e14: smithi089.srlyxf(active, since 42s) 2024-09-21T04:03:09.264 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:03:08 smithi089 ceph-mon[23733]: from='client.? 172.21.15.195:0/2076793798' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-21T04:03:09.629 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-21T04:03:09.630 DEBUG:teuthology.orchestra.run.smithi195:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 1b9b79fc-77ce-11ef-baf5-efdc52797490 -- ceph mon dump -f json 2024-09-21T04:03:11.411 INFO:teuthology.orchestra.run.smithi195.stdout: 2024-09-21T04:03:11.412 INFO:teuthology.orchestra.run.smithi195.stdout:{"epoch":1,"fsid":"1b9b79fc-77ce-11ef-baf5-efdc52797490","modified":"2024-09-21T04:01:28.028960Z","created":"2024-09-21T04:01:28.028960Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi089","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:3300","nonce":0},{"type":"v1","addr":"172.21.15.89:6789","nonce":0}]},"addr":"172.21.15.89:6789/0","public_addr":"172.21.15.89:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-21T04:03:11.413 INFO:teuthology.orchestra.run.smithi195.stderr:dumped monmap epoch 1 2024-09-21T04:03:11.764 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:03:11 smithi089 ceph-mon[23733]: from='client.? 172.21.15.195:0/3344091776' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-21T04:03:12.931 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-21T04:03:12.931 DEBUG:teuthology.orchestra.run.smithi195:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 1b9b79fc-77ce-11ef-baf5-efdc52797490 -- ceph mon dump -f json 2024-09-21T04:03:13.344 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:03:13 smithi089 ceph-mon[23733]: Active manager daemon smithi089.srlyxf restarted 2024-09-21T04:03:13.344 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:03:13 smithi089 ceph-mon[23733]: Activating manager daemon smithi089.srlyxf 2024-09-21T04:03:13.344 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:03:13 smithi089 ceph-mon[23733]: osdmap e5: 0 total, 0 up, 0 in 2024-09-21T04:03:14.515 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:03:14 smithi089 ceph-mon[23733]: mgrmap e15: smithi089.srlyxf(active, starting, since 0.0300763s) 2024-09-21T04:03:14.515 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:03:14 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "mon metadata", "id": "smithi089"}]: dispatch 2024-09-21T04:03:14.515 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:03:14 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "mgr metadata", "who": "smithi089.srlyxf", "id": "smithi089.srlyxf"}]: dispatch 2024-09-21T04:03:14.515 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:03:14 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-21T04:03:14.515 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:03:14 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-21T04:03:14.515 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:03:14 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-21T04:03:14.515 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:03:14 smithi089 ceph-mon[23733]: Manager daemon smithi089.srlyxf is now available 2024-09-21T04:03:14.515 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:03:14 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T04:03:14.515 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:03:14 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T04:03:14.515 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:03:14 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T04:03:14.515 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:03:14 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T04:03:14.516 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:03:14 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' 2024-09-21T04:03:14.516 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:03:14 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi089.srlyxf/mirror_snapshot_schedule"}]: dispatch 2024-09-21T04:03:14.516 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:03:14 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi089.srlyxf/trash_purge_schedule"}]: dispatch 2024-09-21T04:03:14.516 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:03:14 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' 2024-09-21T04:03:14.516 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:03:14 smithi089 ceph-mon[23733]: [21/Sep/2024:04:03:13] ENGINE Bus STARTING 2024-09-21T04:03:14.516 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:03:14 smithi089 ceph-mon[23733]: [21/Sep/2024:04:03:13] ENGINE Serving on https://172.21.15.89:7150 2024-09-21T04:03:14.516 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:03:14 smithi089 ceph-mon[23733]: [21/Sep/2024:04:03:13] ENGINE Bus STARTED 2024-09-21T04:03:14.516 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:03:14 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' 2024-09-21T04:03:14.824 INFO:teuthology.orchestra.run.smithi195.stdout: 2024-09-21T04:03:14.824 INFO:teuthology.orchestra.run.smithi195.stdout:{"epoch":1,"fsid":"1b9b79fc-77ce-11ef-baf5-efdc52797490","modified":"2024-09-21T04:01:28.028960Z","created":"2024-09-21T04:01:28.028960Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi089","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:3300","nonce":0},{"type":"v1","addr":"172.21.15.89:6789","nonce":0}]},"addr":"172.21.15.89:6789/0","public_addr":"172.21.15.89:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-21T04:03:14.826 INFO:teuthology.orchestra.run.smithi195.stderr:dumped monmap epoch 1 2024-09-21T04:03:15.514 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:03:15 smithi089 ceph-mon[23733]: mgrmap e16: smithi089.srlyxf(active, since 1.06479s) 2024-09-21T04:03:15.514 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:03:15 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' 2024-09-21T04:03:15.515 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:03:15 smithi089 ceph-mon[23733]: from='client.? 172.21.15.195:0/1800040135' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-21T04:03:16.469 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-21T04:03:16.469 DEBUG:teuthology.orchestra.run.smithi195:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 1b9b79fc-77ce-11ef-baf5-efdc52797490 -- ceph mon dump -f json 2024-09-21T04:03:16.764 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:03:16 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' 2024-09-21T04:03:16.764 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:03:16 smithi089 ceph-mon[23733]: mgrmap e17: smithi089.srlyxf(active, since 2s) 2024-09-21T04:03:19.215 INFO:teuthology.orchestra.run.smithi195.stdout: 2024-09-21T04:03:19.215 INFO:teuthology.orchestra.run.smithi195.stdout:{"epoch":1,"fsid":"1b9b79fc-77ce-11ef-baf5-efdc52797490","modified":"2024-09-21T04:01:28.028960Z","created":"2024-09-21T04:01:28.028960Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi089","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:3300","nonce":0},{"type":"v1","addr":"172.21.15.89:6789","nonce":0}]},"addr":"172.21.15.89:6789/0","public_addr":"172.21.15.89:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-21T04:03:19.217 INFO:teuthology.orchestra.run.smithi195.stderr:dumped monmap epoch 1 2024-09-21T04:03:19.514 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:03:19 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' 2024-09-21T04:03:19.514 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:03:19 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' 2024-09-21T04:03:19.515 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:03:19 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "config rm", "who": "osd/host:smithi195", "name": "osd_memory_target"}]: dispatch 2024-09-21T04:03:20.427 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:03:20 smithi089 ceph-mon[23733]: from='client.? 172.21.15.195:0/866138238' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-21T04:03:20.427 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:03:20 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' 2024-09-21T04:03:20.427 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:03:20 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' 2024-09-21T04:03:20.428 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:03:20 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "config rm", "who": "osd/host:smithi089", "name": "osd_memory_target"}]: dispatch 2024-09-21T04:03:20.742 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-21T04:03:20.743 DEBUG:teuthology.orchestra.run.smithi195:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 1b9b79fc-77ce-11ef-baf5-efdc52797490 -- ceph mon dump -f json 2024-09-21T04:03:21.765 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:03:21 smithi089 ceph-mon[23733]: Updating smithi089:/etc/ceph/ceph.conf 2024-09-21T04:03:21.765 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:03:21 smithi089 ceph-mon[23733]: Updating smithi089:/etc/ceph/ceph.client.admin.keyring 2024-09-21T04:03:21.765 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:03:21 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' 2024-09-21T04:03:21.765 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:03:21 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' 2024-09-21T04:03:21.765 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:03:21 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi195", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-09-21T04:03:21.765 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:03:21 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd='[{"prefix": "auth get-or-create", "entity": "client.crash.smithi195", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]': finished 2024-09-21T04:03:21.765 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:03:21 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T04:03:21.765 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:03:21 smithi089 ceph-mon[23733]: Deploying daemon crash.smithi195 on smithi195 2024-09-21T04:03:23.549 INFO:teuthology.orchestra.run.smithi195.stdout: 2024-09-21T04:03:23.549 INFO:teuthology.orchestra.run.smithi195.stdout:{"epoch":1,"fsid":"1b9b79fc-77ce-11ef-baf5-efdc52797490","modified":"2024-09-21T04:01:28.028960Z","created":"2024-09-21T04:01:28.028960Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi089","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:3300","nonce":0},{"type":"v1","addr":"172.21.15.89:6789","nonce":0}]},"addr":"172.21.15.89:6789/0","public_addr":"172.21.15.89:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-21T04:03:23.551 INFO:teuthology.orchestra.run.smithi195.stderr:dumped monmap epoch 1 2024-09-21T04:03:23.852 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:03:23 smithi089 ceph-mon[23733]: from='client.? 172.21.15.195:0/3952585921' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-21T04:03:25.173 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-21T04:03:25.173 DEBUG:teuthology.orchestra.run.smithi195:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 1b9b79fc-77ce-11ef-baf5-efdc52797490 -- ceph mon dump -f json 2024-09-21T04:03:25.764 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:03:25 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' 2024-09-21T04:03:25.764 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:03:25 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi195.cgotzm", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-21T04:03:25.764 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:03:25 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.smithi195.cgotzm", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2024-09-21T04:03:25.764 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:03:25 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-21T04:03:25.764 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:03:25 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T04:03:25.765 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:03:25 smithi089 ceph-mon[23733]: Deploying daemon mgr.smithi195.cgotzm on smithi195 2024-09-21T04:03:27.913 INFO:teuthology.orchestra.run.smithi195.stdout: 2024-09-21T04:03:27.913 INFO:teuthology.orchestra.run.smithi195.stdout:{"epoch":1,"fsid":"1b9b79fc-77ce-11ef-baf5-efdc52797490","modified":"2024-09-21T04:01:28.028960Z","created":"2024-09-21T04:01:28.028960Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi089","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:3300","nonce":0},{"type":"v1","addr":"172.21.15.89:6789","nonce":0}]},"addr":"172.21.15.89:6789/0","public_addr":"172.21.15.89:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-21T04:03:27.915 INFO:teuthology.orchestra.run.smithi195.stderr:dumped monmap epoch 1 2024-09-21T04:03:29.264 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:03:28 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' 2024-09-21T04:03:29.265 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:03:28 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-21T04:03:29.265 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:03:28 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T04:03:29.265 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:03:28 smithi089 ceph-mon[23733]: Deploying daemon mon.smithi195 on smithi195 2024-09-21T04:03:29.265 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:03:28 smithi089 ceph-mon[23733]: from='client.? 172.21.15.195:0/3801632101' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-21T04:03:29.265 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:03:28 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' 2024-09-21T04:03:29.528 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-21T04:03:29.529 DEBUG:teuthology.orchestra.run.smithi195:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 1b9b79fc-77ce-11ef-baf5-efdc52797490 -- ceph mon dump -f json 2024-09-21T04:03:32.382 INFO:teuthology.orchestra.run.smithi195.stdout: 2024-09-21T04:03:32.382 INFO:teuthology.orchestra.run.smithi195.stdout:{"epoch":1,"fsid":"1b9b79fc-77ce-11ef-baf5-efdc52797490","modified":"2024-09-21T04:01:28.028960Z","created":"2024-09-21T04:01:28.028960Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi089","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:3300","nonce":0},{"type":"v1","addr":"172.21.15.89:6789","nonce":0}]},"addr":"172.21.15.89:6789/0","public_addr":"172.21.15.89:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-21T04:03:32.383 INFO:teuthology.orchestra.run.smithi195.stderr:dumped monmap epoch 1 2024-09-21T04:03:34.207 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-21T04:03:34.207 DEBUG:teuthology.orchestra.run.smithi195:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 1b9b79fc-77ce-11ef-baf5-efdc52797490 -- ceph mon dump -f json 2024-09-21T04:03:38.265 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:03:37 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "mon metadata", "id": "smithi089"}]: dispatch 2024-09-21T04:03:38.265 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:03:37 smithi089 ceph-mon[23733]: mon.smithi089 calling monitor election 2024-09-21T04:03:38.265 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:03:37 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "mon metadata", "id": "smithi195"}]: dispatch 2024-09-21T04:03:38.265 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:03:37 smithi089 ceph-mon[23733]: pgmap v3: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T04:03:38.265 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:03:37 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "mon metadata", "id": "smithi195"}]: dispatch 2024-09-21T04:03:38.265 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:03:37 smithi089 ceph-mon[23733]: from='mgr.? 172.21.15.195:0/1728897276' entity='mgr.smithi195.cgotzm' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi195.cgotzm/crt"}]: dispatch 2024-09-21T04:03:38.265 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:03:37 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "mon metadata", "id": "smithi195"}]: dispatch 2024-09-21T04:03:38.265 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:03:37 smithi089 ceph-mon[23733]: mon.smithi195 calling monitor election 2024-09-21T04:03:38.265 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:03:37 smithi089 ceph-mon[23733]: pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T04:03:38.265 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:03:37 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "mon metadata", "id": "smithi195"}]: dispatch 2024-09-21T04:03:38.265 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:03:37 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "mon metadata", "id": "smithi195"}]: dispatch 2024-09-21T04:03:38.265 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:03:37 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "mon metadata", "id": "smithi195"}]: dispatch 2024-09-21T04:03:38.265 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:03:37 smithi089 ceph-mon[23733]: mon.smithi089 is new leader, mons smithi089,smithi195 in quorum (ranks 0,1) 2024-09-21T04:03:38.266 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:03:37 smithi089 ceph-mon[23733]: monmap e2: 2 mons at {smithi089=[v2:172.21.15.89:3300/0,v1:172.21.15.89:6789/0],smithi195=[v2:172.21.15.195:3300/0,v1:172.21.15.195:6789/0]} 2024-09-21T04:03:38.266 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:03:37 smithi089 ceph-mon[23733]: fsmap 2024-09-21T04:03:38.266 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:03:37 smithi089 ceph-mon[23733]: osdmap e5: 0 total, 0 up, 0 in 2024-09-21T04:03:38.266 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:03:37 smithi089 ceph-mon[23733]: mgrmap e17: smithi089.srlyxf(active, since 24s) 2024-09-21T04:03:38.266 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:03:37 smithi089 ceph-mon[23733]: Standby manager daemon smithi195.cgotzm started 2024-09-21T04:03:38.266 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:03:37 smithi089 ceph-mon[23733]: from='mgr.? 172.21.15.195:0/1728897276' entity='mgr.smithi195.cgotzm' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-21T04:03:38.266 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:03:37 smithi089 ceph-mon[23733]: overall HEALTH_OK 2024-09-21T04:03:38.266 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:03:37 smithi089 ceph-mon[23733]: from='mgr.? 172.21.15.195:0/1728897276' entity='mgr.smithi195.cgotzm' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi195.cgotzm/key"}]: dispatch 2024-09-21T04:03:38.266 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:03:37 smithi089 ceph-mon[23733]: from='mgr.? 172.21.15.195:0/1728897276' entity='mgr.smithi195.cgotzm' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-21T04:03:38.266 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:03:37 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' 2024-09-21T04:03:39.011 INFO:teuthology.orchestra.run.smithi195.stdout: 2024-09-21T04:03:39.011 INFO:teuthology.orchestra.run.smithi195.stdout:{"epoch":2,"fsid":"1b9b79fc-77ce-11ef-baf5-efdc52797490","modified":"2024-09-21T04:03:32.716481Z","created":"2024-09-21T04:01:28.028960Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi089","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:3300","nonce":0},{"type":"v1","addr":"172.21.15.89:6789","nonce":0}]},"addr":"172.21.15.89:6789/0","public_addr":"172.21.15.89:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":1,"name":"smithi195","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:3300","nonce":0},{"type":"v1","addr":"172.21.15.195:6789","nonce":0}]},"addr":"172.21.15.195:6789/0","public_addr":"172.21.15.195:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0,1]} 2024-09-21T04:03:39.012 INFO:teuthology.orchestra.run.smithi195.stderr:dumped monmap epoch 2 2024-09-21T04:03:39.264 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:03:38 smithi089 ceph-mon[23733]: pgmap v5: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T04:03:39.265 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:03:38 smithi089 ceph-mon[23733]: Deploying daemon node-exporter.smithi195 on smithi195 2024-09-21T04:03:39.265 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:03:38 smithi089 ceph-mon[23733]: mgrmap e18: smithi089.srlyxf(active, since 24s), standbys: smithi195.cgotzm 2024-09-21T04:03:39.265 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:03:38 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "mgr metadata", "who": "smithi195.cgotzm", "id": "smithi195.cgotzm"}]: dispatch 2024-09-21T04:03:39.265 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:03:38 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' 2024-09-21T04:03:39.265 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:03:38 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "mon metadata", "id": "smithi195"}]: dispatch 2024-09-21T04:03:40.108 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:03:39 smithi195 ceph-mon[25840]: from='client.? 172.21.15.195:0/1516101114' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-21T04:03:40.264 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:03:39 smithi089 ceph-mon[23733]: from='client.? 172.21.15.195:0/1516101114' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-21T04:03:40.382 INFO:tasks.cephadm:Generating final ceph.conf file... 2024-09-21T04:03:40.382 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 1b9b79fc-77ce-11ef-baf5-efdc52797490 -- ceph config generate-minimal-conf 2024-09-21T04:03:40.979 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:03:40 smithi195 ceph-mon[25840]: pgmap v6: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T04:03:41.006 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:03:40 smithi089 ceph-mon[23733]: pgmap v6: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T04:03:42.275 INFO:teuthology.orchestra.run.smithi089.stdout:# minimal ceph.conf for 1b9b79fc-77ce-11ef-baf5-efdc52797490 2024-09-21T04:03:42.276 INFO:teuthology.orchestra.run.smithi089.stdout:[global] 2024-09-21T04:03:42.276 INFO:teuthology.orchestra.run.smithi089.stdout: fsid = 1b9b79fc-77ce-11ef-baf5-efdc52797490 2024-09-21T04:03:42.276 INFO:teuthology.orchestra.run.smithi089.stdout: mon_host = [v2:172.21.15.89:3300/0,v1:172.21.15.89:6789/0] [v2:172.21.15.195:3300/0,v1:172.21.15.195:6789/0] 2024-09-21T04:03:42.608 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:03:42 smithi195 ceph-mon[25840]: pgmap v7: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T04:03:42.608 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:03:42 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' 2024-09-21T04:03:42.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:03:42 smithi195 ceph-mon[25840]: Deploying daemon prometheus.smithi089 on smithi089 2024-09-21T04:03:42.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:03:42 smithi195 ceph-mon[25840]: from='client.? 172.21.15.89:0/2416927070' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T04:03:42.698 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:03:42 smithi089 ceph-mon[23733]: pgmap v7: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T04:03:42.698 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:03:42 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' 2024-09-21T04:03:42.698 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:03:42 smithi089 ceph-mon[23733]: Deploying daemon prometheus.smithi089 on smithi089 2024-09-21T04:03:42.699 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:03:42 smithi089 ceph-mon[23733]: from='client.? 172.21.15.89:0/2416927070' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T04:03:43.091 INFO:tasks.cephadm:Distributing (final) config and client.admin keyring... 2024-09-21T04:03:43.092 DEBUG:teuthology.orchestra.run.smithi089:> set -ex 2024-09-21T04:03:43.092 DEBUG:teuthology.orchestra.run.smithi089:> sudo dd of=/etc/ceph/ceph.conf 2024-09-21T04:03:43.120 DEBUG:teuthology.orchestra.run.smithi089:> set -ex 2024-09-21T04:03:43.120 DEBUG:teuthology.orchestra.run.smithi089:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2024-09-21T04:03:43.184 DEBUG:teuthology.orchestra.run.smithi195:> set -ex 2024-09-21T04:03:43.185 DEBUG:teuthology.orchestra.run.smithi195:> sudo dd of=/etc/ceph/ceph.conf 2024-09-21T04:03:43.217 DEBUG:teuthology.orchestra.run.smithi195:> set -ex 2024-09-21T04:03:43.218 DEBUG:teuthology.orchestra.run.smithi195:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2024-09-21T04:03:43.286 DEBUG:tasks.cephadm:set 0 configs 2024-09-21T04:03:43.287 INFO:tasks.cephadm:Deploying OSDs... 2024-09-21T04:03:43.287 DEBUG:teuthology.orchestra.run.smithi089:> set -ex 2024-09-21T04:03:43.287 DEBUG:teuthology.orchestra.run.smithi089:> dd if=/scratch_devs of=/dev/stdout 2024-09-21T04:03:43.303 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2024-09-21T04:03:43.303 DEBUG:teuthology.orchestra.run.smithi089:> stat /dev/vg_nvme/lv_1 2024-09-21T04:03:43.359 INFO:teuthology.orchestra.run.smithi089.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2024-09-21T04:03:43.359 INFO:teuthology.orchestra.run.smithi089.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-21T04:03:43.359 INFO:teuthology.orchestra.run.smithi089.stdout:Device: 5h/5d Inode: 668 Links: 1 2024-09-21T04:03:43.359 INFO:teuthology.orchestra.run.smithi089.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-21T04:03:43.359 INFO:teuthology.orchestra.run.smithi089.stdout:Context: system_u:object_r:device_t:s0 2024-09-21T04:03:43.359 INFO:teuthology.orchestra.run.smithi089.stdout:Access: 2024-09-21 04:03:03.634638706 +0000 2024-09-21T04:03:43.359 INFO:teuthology.orchestra.run.smithi089.stdout:Modify: 2024-09-21 04:00:37.437501752 +0000 2024-09-21T04:03:43.359 INFO:teuthology.orchestra.run.smithi089.stdout:Change: 2024-09-21 04:00:37.437501752 +0000 2024-09-21T04:03:43.359 INFO:teuthology.orchestra.run.smithi089.stdout: Birth: 2024-09-21 04:00:37.437501752 +0000 2024-09-21T04:03:43.360 DEBUG:teuthology.orchestra.run.smithi089:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2024-09-21T04:03:43.423 INFO:teuthology.orchestra.run.smithi089.stderr:1+0 records in 2024-09-21T04:03:43.423 INFO:teuthology.orchestra.run.smithi089.stderr:1+0 records out 2024-09-21T04:03:43.423 INFO:teuthology.orchestra.run.smithi089.stderr:512 bytes copied, 0.000206331 s, 2.5 MB/s 2024-09-21T04:03:43.424 DEBUG:teuthology.orchestra.run.smithi089:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2024-09-21T04:03:43.482 DEBUG:teuthology.orchestra.run.smithi089:> stat /dev/vg_nvme/lv_2 2024-09-21T04:03:43.541 INFO:teuthology.orchestra.run.smithi089.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2024-09-21T04:03:43.541 INFO:teuthology.orchestra.run.smithi089.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-21T04:03:43.541 INFO:teuthology.orchestra.run.smithi089.stdout:Device: 5h/5d Inode: 686 Links: 1 2024-09-21T04:03:43.541 INFO:teuthology.orchestra.run.smithi089.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-21T04:03:43.541 INFO:teuthology.orchestra.run.smithi089.stdout:Context: system_u:object_r:device_t:s0 2024-09-21T04:03:43.541 INFO:teuthology.orchestra.run.smithi089.stdout:Access: 2024-09-21 04:03:03.634638706 +0000 2024-09-21T04:03:43.541 INFO:teuthology.orchestra.run.smithi089.stdout:Modify: 2024-09-21 04:00:37.767494136 +0000 2024-09-21T04:03:43.541 INFO:teuthology.orchestra.run.smithi089.stdout:Change: 2024-09-21 04:00:37.767494136 +0000 2024-09-21T04:03:43.541 INFO:teuthology.orchestra.run.smithi089.stdout: Birth: 2024-09-21 04:00:37.767494136 +0000 2024-09-21T04:03:43.542 DEBUG:teuthology.orchestra.run.smithi089:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2024-09-21T04:03:43.609 INFO:teuthology.orchestra.run.smithi089.stderr:1+0 records in 2024-09-21T04:03:43.609 INFO:teuthology.orchestra.run.smithi089.stderr:1+0 records out 2024-09-21T04:03:43.609 INFO:teuthology.orchestra.run.smithi089.stderr:512 bytes copied, 0.000172676 s, 3.0 MB/s 2024-09-21T04:03:43.610 DEBUG:teuthology.orchestra.run.smithi089:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2024-09-21T04:03:43.669 DEBUG:teuthology.orchestra.run.smithi089:> stat /dev/vg_nvme/lv_3 2024-09-21T04:03:43.728 INFO:teuthology.orchestra.run.smithi089.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2024-09-21T04:03:43.728 INFO:teuthology.orchestra.run.smithi089.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-21T04:03:43.728 INFO:teuthology.orchestra.run.smithi089.stdout:Device: 5h/5d Inode: 706 Links: 1 2024-09-21T04:03:43.729 INFO:teuthology.orchestra.run.smithi089.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-21T04:03:43.729 INFO:teuthology.orchestra.run.smithi089.stdout:Context: system_u:object_r:device_t:s0 2024-09-21T04:03:43.729 INFO:teuthology.orchestra.run.smithi089.stdout:Access: 2024-09-21 04:03:03.635638683 +0000 2024-09-21T04:03:43.729 INFO:teuthology.orchestra.run.smithi089.stdout:Modify: 2024-09-21 04:00:38.091486659 +0000 2024-09-21T04:03:43.729 INFO:teuthology.orchestra.run.smithi089.stdout:Change: 2024-09-21 04:00:38.091486659 +0000 2024-09-21T04:03:43.729 INFO:teuthology.orchestra.run.smithi089.stdout: Birth: 2024-09-21 04:00:38.091486659 +0000 2024-09-21T04:03:43.729 DEBUG:teuthology.orchestra.run.smithi089:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2024-09-21T04:03:43.800 INFO:teuthology.orchestra.run.smithi089.stderr:1+0 records in 2024-09-21T04:03:43.800 INFO:teuthology.orchestra.run.smithi089.stderr:1+0 records out 2024-09-21T04:03:43.801 INFO:teuthology.orchestra.run.smithi089.stderr:512 bytes copied, 0.00029156 s, 1.8 MB/s 2024-09-21T04:03:43.802 DEBUG:teuthology.orchestra.run.smithi089:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2024-09-21T04:03:43.865 DEBUG:teuthology.orchestra.run.smithi089:> stat /dev/vg_nvme/lv_4 2024-09-21T04:03:43.921 INFO:teuthology.orchestra.run.smithi089.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2024-09-21T04:03:43.921 INFO:teuthology.orchestra.run.smithi089.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-21T04:03:43.921 INFO:teuthology.orchestra.run.smithi089.stdout:Device: 5h/5d Inode: 723 Links: 1 2024-09-21T04:03:43.921 INFO:teuthology.orchestra.run.smithi089.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-21T04:03:43.922 INFO:teuthology.orchestra.run.smithi089.stdout:Context: system_u:object_r:device_t:s0 2024-09-21T04:03:43.922 INFO:teuthology.orchestra.run.smithi089.stdout:Access: 2024-09-21 04:03:03.635638683 +0000 2024-09-21T04:03:43.922 INFO:teuthology.orchestra.run.smithi089.stdout:Modify: 2024-09-21 04:00:38.414479204 +0000 2024-09-21T04:03:43.922 INFO:teuthology.orchestra.run.smithi089.stdout:Change: 2024-09-21 04:00:38.414479204 +0000 2024-09-21T04:03:43.922 INFO:teuthology.orchestra.run.smithi089.stdout: Birth: 2024-09-21 04:00:38.414479204 +0000 2024-09-21T04:03:43.922 DEBUG:teuthology.orchestra.run.smithi089:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2024-09-21T04:03:43.988 INFO:teuthology.orchestra.run.smithi089.stderr:1+0 records in 2024-09-21T04:03:43.988 INFO:teuthology.orchestra.run.smithi089.stderr:1+0 records out 2024-09-21T04:03:43.988 INFO:teuthology.orchestra.run.smithi089.stderr:512 bytes copied, 0.00017973 s, 2.8 MB/s 2024-09-21T04:03:43.990 DEBUG:teuthology.orchestra.run.smithi089:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2024-09-21T04:03:44.049 DEBUG:teuthology.orchestra.run.smithi195:> set -ex 2024-09-21T04:03:44.049 DEBUG:teuthology.orchestra.run.smithi195:> dd if=/scratch_devs of=/dev/stdout 2024-09-21T04:03:44.068 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2024-09-21T04:03:44.068 DEBUG:teuthology.orchestra.run.smithi195:> stat /dev/vg_nvme/lv_1 2024-09-21T04:03:44.126 INFO:teuthology.orchestra.run.smithi195.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2024-09-21T04:03:44.127 INFO:teuthology.orchestra.run.smithi195.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-21T04:03:44.127 INFO:teuthology.orchestra.run.smithi195.stdout:Device: 5h/5d Inode: 668 Links: 1 2024-09-21T04:03:44.127 INFO:teuthology.orchestra.run.smithi195.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-21T04:03:44.127 INFO:teuthology.orchestra.run.smithi195.stdout:Context: system_u:object_r:device_t:s0 2024-09-21T04:03:44.127 INFO:teuthology.orchestra.run.smithi195.stdout:Access: 2024-09-21 04:03:38.303580563 +0000 2024-09-21T04:03:44.127 INFO:teuthology.orchestra.run.smithi195.stdout:Modify: 2024-09-21 04:00:37.428709203 +0000 2024-09-21T04:03:44.127 INFO:teuthology.orchestra.run.smithi195.stdout:Change: 2024-09-21 04:00:37.428709203 +0000 2024-09-21T04:03:44.127 INFO:teuthology.orchestra.run.smithi195.stdout: Birth: 2024-09-21 04:00:37.428709203 +0000 2024-09-21T04:03:44.127 DEBUG:teuthology.orchestra.run.smithi195:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2024-09-21T04:03:44.193 INFO:teuthology.orchestra.run.smithi195.stderr:1+0 records in 2024-09-21T04:03:44.194 INFO:teuthology.orchestra.run.smithi195.stderr:1+0 records out 2024-09-21T04:03:44.194 INFO:teuthology.orchestra.run.smithi195.stderr:512 bytes copied, 0.000129952 s, 3.9 MB/s 2024-09-21T04:03:44.195 DEBUG:teuthology.orchestra.run.smithi195:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2024-09-21T04:03:44.253 DEBUG:teuthology.orchestra.run.smithi195:> stat /dev/vg_nvme/lv_2 2024-09-21T04:03:44.311 INFO:teuthology.orchestra.run.smithi195.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2024-09-21T04:03:44.311 INFO:teuthology.orchestra.run.smithi195.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-21T04:03:44.311 INFO:teuthology.orchestra.run.smithi195.stdout:Device: 5h/5d Inode: 685 Links: 1 2024-09-21T04:03:44.311 INFO:teuthology.orchestra.run.smithi195.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-21T04:03:44.311 INFO:teuthology.orchestra.run.smithi195.stdout:Context: system_u:object_r:device_t:s0 2024-09-21T04:03:44.311 INFO:teuthology.orchestra.run.smithi195.stdout:Access: 2024-09-21 04:03:38.303580563 +0000 2024-09-21T04:03:44.312 INFO:teuthology.orchestra.run.smithi195.stdout:Modify: 2024-09-21 04:00:37.754702770 +0000 2024-09-21T04:03:44.312 INFO:teuthology.orchestra.run.smithi195.stdout:Change: 2024-09-21 04:00:37.754702770 +0000 2024-09-21T04:03:44.312 INFO:teuthology.orchestra.run.smithi195.stdout: Birth: 2024-09-21 04:00:37.754702770 +0000 2024-09-21T04:03:44.312 DEBUG:teuthology.orchestra.run.smithi195:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2024-09-21T04:03:44.374 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:03:44 smithi195 ceph-mon[25840]: pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T04:03:44.374 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:03:44 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' 2024-09-21T04:03:44.376 INFO:teuthology.orchestra.run.smithi195.stderr:1+0 records in 2024-09-21T04:03:44.376 INFO:teuthology.orchestra.run.smithi195.stderr:1+0 records out 2024-09-21T04:03:44.377 INFO:teuthology.orchestra.run.smithi195.stderr:512 bytes copied, 0.00011879 s, 4.3 MB/s 2024-09-21T04:03:44.378 DEBUG:teuthology.orchestra.run.smithi195:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2024-09-21T04:03:44.436 DEBUG:teuthology.orchestra.run.smithi195:> stat /dev/vg_nvme/lv_3 2024-09-21T04:03:44.491 INFO:teuthology.orchestra.run.smithi195.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2024-09-21T04:03:44.492 INFO:teuthology.orchestra.run.smithi195.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-21T04:03:44.492 INFO:teuthology.orchestra.run.smithi195.stdout:Device: 5h/5d Inode: 706 Links: 1 2024-09-21T04:03:44.492 INFO:teuthology.orchestra.run.smithi195.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-21T04:03:44.492 INFO:teuthology.orchestra.run.smithi195.stdout:Context: system_u:object_r:device_t:s0 2024-09-21T04:03:44.492 INFO:teuthology.orchestra.run.smithi195.stdout:Access: 2024-09-21 04:03:38.304580543 +0000 2024-09-21T04:03:44.492 INFO:teuthology.orchestra.run.smithi195.stdout:Modify: 2024-09-21 04:00:38.059696752 +0000 2024-09-21T04:03:44.492 INFO:teuthology.orchestra.run.smithi195.stdout:Change: 2024-09-21 04:00:38.059696752 +0000 2024-09-21T04:03:44.492 INFO:teuthology.orchestra.run.smithi195.stdout: Birth: 2024-09-21 04:00:38.059696752 +0000 2024-09-21T04:03:44.492 DEBUG:teuthology.orchestra.run.smithi195:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2024-09-21T04:03:44.514 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:03:44 smithi089 ceph-mon[23733]: pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T04:03:44.514 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:03:44 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' 2024-09-21T04:03:44.555 INFO:teuthology.orchestra.run.smithi195.stderr:1+0 records in 2024-09-21T04:03:44.555 INFO:teuthology.orchestra.run.smithi195.stderr:1+0 records out 2024-09-21T04:03:44.555 INFO:teuthology.orchestra.run.smithi195.stderr:512 bytes copied, 9.2528e-05 s, 5.5 MB/s 2024-09-21T04:03:44.556 DEBUG:teuthology.orchestra.run.smithi195:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2024-09-21T04:03:44.612 DEBUG:teuthology.orchestra.run.smithi195:> stat /dev/vg_nvme/lv_4 2024-09-21T04:03:44.667 INFO:teuthology.orchestra.run.smithi195.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2024-09-21T04:03:44.667 INFO:teuthology.orchestra.run.smithi195.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-21T04:03:44.667 INFO:teuthology.orchestra.run.smithi195.stdout:Device: 5h/5d Inode: 723 Links: 1 2024-09-21T04:03:44.667 INFO:teuthology.orchestra.run.smithi195.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-21T04:03:44.668 INFO:teuthology.orchestra.run.smithi195.stdout:Context: system_u:object_r:device_t:s0 2024-09-21T04:03:44.668 INFO:teuthology.orchestra.run.smithi195.stdout:Access: 2024-09-21 04:03:38.304580543 +0000 2024-09-21T04:03:44.668 INFO:teuthology.orchestra.run.smithi195.stdout:Modify: 2024-09-21 04:00:38.378690458 +0000 2024-09-21T04:03:44.668 INFO:teuthology.orchestra.run.smithi195.stdout:Change: 2024-09-21 04:00:38.378690458 +0000 2024-09-21T04:03:44.668 INFO:teuthology.orchestra.run.smithi195.stdout: Birth: 2024-09-21 04:00:38.378690458 +0000 2024-09-21T04:03:44.668 DEBUG:teuthology.orchestra.run.smithi195:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2024-09-21T04:03:44.730 INFO:teuthology.orchestra.run.smithi195.stderr:1+0 records in 2024-09-21T04:03:44.730 INFO:teuthology.orchestra.run.smithi195.stderr:1+0 records out 2024-09-21T04:03:44.731 INFO:teuthology.orchestra.run.smithi195.stderr:512 bytes copied, 0.000114351 s, 4.5 MB/s 2024-09-21T04:03:44.732 DEBUG:teuthology.orchestra.run.smithi195:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2024-09-21T04:03:44.788 INFO:tasks.cephadm:Deploying osd.0 on smithi089 with /dev/vg_nvme/lv_4... 2024-09-21T04:03:44.789 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 1b9b79fc-77ce-11ef-baf5-efdc52797490 -- lvm zap /dev/vg_nvme/lv_4 2024-09-21T04:03:46.514 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:03:46 smithi089 ceph-mon[23733]: pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T04:03:46.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:03:46 smithi195 ceph-mon[25840]: pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T04:03:48.358 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:03:48 smithi195 ceph-mon[25840]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T04:03:48.514 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:03:48 smithi089 ceph-mon[23733]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T04:03:50.416 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:03:50 smithi089 ceph-mon[23733]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T04:03:50.608 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:03:50 smithi195 ceph-mon[25840]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T04:03:52.515 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:03:52 smithi089 ceph-mon[23733]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T04:03:52.608 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:03:52 smithi195 ceph-mon[25840]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T04:03:54.014 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:03:53 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' 2024-09-21T04:03:54.015 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:03:53 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T04:03:54.015 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:03:53 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T04:03:54.015 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:03:53 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T04:03:54.015 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:03:53 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' 2024-09-21T04:03:54.164 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:03:53 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' 2024-09-21T04:03:54.165 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:03:53 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T04:03:54.165 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:03:53 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T04:03:54.165 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:03:53 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T04:03:54.165 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:03:53 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' 2024-09-21T04:03:54.386 INFO:teuthology.orchestra.run.smithi089.stdout: 2024-09-21T04:03:54.404 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 1b9b79fc-77ce-11ef-baf5-efdc52797490 -- ceph orch daemon add osd smithi089:vg_nvme/lv_4 2024-09-21T04:03:55.108 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:03:54 smithi195 ceph-mon[25840]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T04:03:55.108 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:03:54 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' 2024-09-21T04:03:55.135 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:03:54 smithi089 ceph-mon[23733]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T04:03:55.135 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:03:54 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' 2024-09-21T04:03:56.264 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:03:55 smithi089 ceph-mon[23733]: Updating smithi195:/etc/ceph/ceph.conf 2024-09-21T04:03:56.264 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:03:55 smithi089 ceph-mon[23733]: Updating smithi195:/etc/ceph/ceph.client.admin.keyring 2024-09-21T04:03:56.264 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:03:55 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' 2024-09-21T04:03:56.358 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:03:55 smithi195 ceph-mon[25840]: Updating smithi195:/etc/ceph/ceph.conf 2024-09-21T04:03:56.358 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:03:55 smithi195 ceph-mon[25840]: Updating smithi195:/etc/ceph/ceph.client.admin.keyring 2024-09-21T04:03:56.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:03:55 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' 2024-09-21T04:03:57.233 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:03:56 smithi089 ceph-mon[23733]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T04:03:57.358 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:03:56 smithi195 ceph-mon[25840]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T04:03:58.338 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:03:58 smithi089 ceph-mon[23733]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T04:03:58.338 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:03:58 smithi089 ceph-mon[23733]: from='client.14234 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi089:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T04:03:58.338 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:03:58 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-21T04:03:58.338 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:03:58 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-21T04:03:58.339 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:03:58 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T04:03:58.358 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:03:58 smithi195 ceph-mon[25840]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T04:03:58.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:03:58 smithi195 ceph-mon[25840]: from='client.14234 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi089:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T04:03:58.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:03:58 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-21T04:03:58.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:03:58 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-21T04:03:58.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:03:58 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T04:04:00.514 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:04:00 smithi089 ceph-mon[23733]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T04:04:00.596 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:04:00 smithi195 ceph-mon[25840]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T04:04:01.514 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:04:01 smithi089 ceph-mon[23733]: from='client.? 172.21.15.89:0/2490415871' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "cb916820-cc81-471e-82a5-0ac43bae0af0"}]: dispatch 2024-09-21T04:04:01.515 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:04:01 smithi089 ceph-mon[23733]: from='client.? 172.21.15.89:0/2490415871' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "cb916820-cc81-471e-82a5-0ac43bae0af0"}]': finished 2024-09-21T04:04:01.515 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:04:01 smithi089 ceph-mon[23733]: osdmap e6: 1 total, 0 up, 1 in 2024-09-21T04:04:01.515 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:04:01 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-21T04:04:01.515 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:04:01 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' 2024-09-21T04:04:01.515 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:04:01 smithi089 ceph-mon[23733]: Updating smithi089:/etc/ceph/ceph.conf 2024-09-21T04:04:01.515 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:04:01 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' 2024-09-21T04:04:01.515 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:04:01 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' 2024-09-21T04:04:01.515 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:04:01 smithi089 ceph-mon[23733]: Reconfiguring mon.smithi089 (unknown last config time)... 2024-09-21T04:04:01.515 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:04:01 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-21T04:04:01.515 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:04:01 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-21T04:04:01.515 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:04:01 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T04:04:01.515 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:04:01 smithi089 ceph-mon[23733]: Reconfiguring daemon mon.smithi089 on smithi089 2024-09-21T04:04:01.515 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:04:01 smithi089 ceph-mon[23733]: from='client.? 172.21.15.89:0/3422200942' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-21T04:04:01.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:04:01 smithi195 ceph-mon[25840]: from='client.? 172.21.15.89:0/2490415871' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "cb916820-cc81-471e-82a5-0ac43bae0af0"}]: dispatch 2024-09-21T04:04:01.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:04:01 smithi195 ceph-mon[25840]: from='client.? 172.21.15.89:0/2490415871' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "cb916820-cc81-471e-82a5-0ac43bae0af0"}]': finished 2024-09-21T04:04:01.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:04:01 smithi195 ceph-mon[25840]: osdmap e6: 1 total, 0 up, 1 in 2024-09-21T04:04:01.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:04:01 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-21T04:04:01.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:04:01 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' 2024-09-21T04:04:01.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:04:01 smithi195 ceph-mon[25840]: Updating smithi089:/etc/ceph/ceph.conf 2024-09-21T04:04:01.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:04:01 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' 2024-09-21T04:04:01.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:04:01 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' 2024-09-21T04:04:01.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:04:01 smithi195 ceph-mon[25840]: Reconfiguring mon.smithi089 (unknown last config time)... 2024-09-21T04:04:01.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:04:01 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-21T04:04:01.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:04:01 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-21T04:04:01.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:04:01 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T04:04:01.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:04:01 smithi195 ceph-mon[25840]: Reconfiguring daemon mon.smithi089 on smithi089 2024-09-21T04:04:01.610 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:04:01 smithi195 ceph-mon[25840]: from='client.? 172.21.15.89:0/3422200942' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-21T04:04:03.108 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:04:02 smithi195 ceph-mon[25840]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T04:04:03.265 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:04:02 smithi089 ceph-mon[23733]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T04:04:04.973 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:04:04 smithi089 ceph-mon[23733]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T04:04:05.108 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:04:04 smithi195 ceph-mon[25840]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T04:04:06.863 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:04:06 smithi089 ceph-mon[23733]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T04:04:06.863 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:04:06 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' 2024-09-21T04:04:06.863 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:04:06 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi089.srlyxf", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-21T04:04:06.863 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:04:06 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-21T04:04:06.863 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:04:06 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T04:04:07.075 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:04:06 smithi195 ceph-mon[25840]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T04:04:07.075 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:04:06 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' 2024-09-21T04:04:07.075 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:04:06 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi089.srlyxf", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-21T04:04:07.075 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:04:06 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-21T04:04:07.075 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:04:06 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T04:04:08.070 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:04:07 smithi089 ceph-mon[23733]: Reconfiguring mgr.smithi089.srlyxf (unknown last config time)... 2024-09-21T04:04:08.070 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:04:07 smithi089 ceph-mon[23733]: Reconfiguring daemon mgr.smithi089.srlyxf on smithi089 2024-09-21T04:04:08.108 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:04:07 smithi195 ceph-mon[25840]: Reconfiguring mgr.smithi089.srlyxf (unknown last config time)... 2024-09-21T04:04:08.109 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:04:07 smithi195 ceph-mon[25840]: Reconfiguring daemon mgr.smithi089.srlyxf on smithi089 2024-09-21T04:04:09.014 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:04:08 smithi089 ceph-mon[23733]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T04:04:09.015 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:04:08 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-09-21T04:04:09.015 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:04:08 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T04:04:09.108 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:04:08 smithi195 ceph-mon[25840]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T04:04:09.108 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:04:08 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-09-21T04:04:09.108 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:04:08 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T04:04:10.108 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:04:09 smithi195 ceph-mon[25840]: Deploying daemon osd.0 on smithi089 2024-09-21T04:04:10.109 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:04:09 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' 2024-09-21T04:04:10.264 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:04:09 smithi089 ceph-mon[23733]: Deploying daemon osd.0 on smithi089 2024-09-21T04:04:10.264 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:04:09 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' 2024-09-21T04:04:10.859 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:04:10 smithi195 ceph-mon[25840]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T04:04:10.859 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:04:10 smithi195 ceph-mon[25840]: Reconfiguring alertmanager.smithi089 (dependencies changed)... 2024-09-21T04:04:10.859 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:04:10 smithi195 ceph-mon[25840]: Reconfiguring daemon alertmanager.smithi089 on smithi089 2024-09-21T04:04:10.927 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:04:10 smithi089 ceph-mon[23733]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T04:04:10.927 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:04:10 smithi089 ceph-mon[23733]: Reconfiguring alertmanager.smithi089 (dependencies changed)... 2024-09-21T04:04:10.928 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:04:10 smithi089 ceph-mon[23733]: Reconfiguring daemon alertmanager.smithi089 on smithi089 2024-09-21T04:04:13.108 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:04:12 smithi195 ceph-mon[25840]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T04:04:13.176 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:04:12 smithi089 ceph-mon[23733]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T04:04:13.865 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:04:13 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi089.srlyxf/mirror_snapshot_schedule"}]: dispatch 2024-09-21T04:04:13.865 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:04:13 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi089.srlyxf/trash_purge_schedule"}]: dispatch 2024-09-21T04:04:14.108 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:04:13 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi089.srlyxf/mirror_snapshot_schedule"}]: dispatch 2024-09-21T04:04:14.109 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:04:13 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi089.srlyxf/trash_purge_schedule"}]: dispatch 2024-09-21T04:04:15.089 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:04:14 smithi089 ceph-mon[23733]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T04:04:15.089 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:04:14 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' 2024-09-21T04:04:15.108 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:04:14 smithi195 ceph-mon[25840]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T04:04:15.108 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:04:14 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' 2024-09-21T04:04:16.858 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:04:16 smithi195 ceph-mon[25840]: pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T04:04:17.134 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:04:16 smithi089 ceph-mon[23733]: pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T04:04:18.341 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:04:18 smithi089 ceph-mon[23733]: pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T04:04:18.341 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:04:18 smithi089 ceph-mon[23733]: from='osd.0 [v2:172.21.15.89:6802/2758701754,v1:172.21.15.89:6803/2758701754]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-09-21T04:04:18.358 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:04:18 smithi195 ceph-mon[25840]: pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T04:04:18.358 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:04:18 smithi195 ceph-mon[25840]: from='osd.0 [v2:172.21.15.89:6802/2758701754,v1:172.21.15.89:6803/2758701754]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-09-21T04:04:19.264 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:04:19 smithi089 ceph-mon[23733]: from='osd.0 [v2:172.21.15.89:6802/2758701754,v1:172.21.15.89:6803/2758701754]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-09-21T04:04:19.264 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:04:19 smithi089 ceph-mon[23733]: osdmap e7: 1 total, 0 up, 1 in 2024-09-21T04:04:19.264 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:04:19 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-21T04:04:19.264 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:04:19 smithi089 ceph-mon[23733]: from='osd.0 [v2:172.21.15.89:6802/2758701754,v1:172.21.15.89:6803/2758701754]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi089", "root=default"]}]: dispatch 2024-09-21T04:04:19.358 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:04:19 smithi195 ceph-mon[25840]: from='osd.0 [v2:172.21.15.89:6802/2758701754,v1:172.21.15.89:6803/2758701754]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-09-21T04:04:19.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:04:19 smithi195 ceph-mon[25840]: osdmap e7: 1 total, 0 up, 1 in 2024-09-21T04:04:19.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:04:19 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-21T04:04:19.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:04:19 smithi195 ceph-mon[25840]: from='osd.0 [v2:172.21.15.89:6802/2758701754,v1:172.21.15.89:6803/2758701754]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi089", "root=default"]}]: dispatch 2024-09-21T04:04:20.265 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:04:20 smithi089 ceph-mon[23733]: pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T04:04:20.265 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:04:20 smithi089 ceph-mon[23733]: from='osd.0 [v2:172.21.15.89:6802/2758701754,v1:172.21.15.89:6803/2758701754]' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi089", "root=default"]}]': finished 2024-09-21T04:04:20.265 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:04:20 smithi089 ceph-mon[23733]: osdmap e8: 1 total, 0 up, 1 in 2024-09-21T04:04:20.265 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:04:20 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-21T04:04:20.265 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:04:20 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-21T04:04:20.265 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:04:20 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' 2024-09-21T04:04:20.265 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:04:20 smithi089 ceph-mon[23733]: Reconfiguring crash.smithi089 (monmap changed)... 2024-09-21T04:04:20.265 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:04:20 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi089", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-09-21T04:04:20.265 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:04:20 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T04:04:20.266 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:04:20 smithi089 ceph-mon[23733]: Reconfiguring daemon crash.smithi089 on smithi089 2024-09-21T04:04:20.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:04:20 smithi195 ceph-mon[25840]: pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T04:04:20.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:04:20 smithi195 ceph-mon[25840]: from='osd.0 [v2:172.21.15.89:6802/2758701754,v1:172.21.15.89:6803/2758701754]' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi089", "root=default"]}]': finished 2024-09-21T04:04:20.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:04:20 smithi195 ceph-mon[25840]: osdmap e8: 1 total, 0 up, 1 in 2024-09-21T04:04:20.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:04:20 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-21T04:04:20.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:04:20 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-21T04:04:20.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:04:20 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' 2024-09-21T04:04:20.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:04:20 smithi195 ceph-mon[25840]: Reconfiguring crash.smithi089 (monmap changed)... 2024-09-21T04:04:20.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:04:20 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi089", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-09-21T04:04:20.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:04:20 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T04:04:20.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:04:20 smithi195 ceph-mon[25840]: Reconfiguring daemon crash.smithi089 on smithi089 2024-09-21T04:04:21.265 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:04:21 smithi089 ceph-mon[23733]: purged_snaps scrub starts 2024-09-21T04:04:21.265 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:04:21 smithi089 ceph-mon[23733]: purged_snaps scrub ok 2024-09-21T04:04:21.265 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:04:21 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-21T04:04:21.265 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:04:21 smithi089 ceph-mon[23733]: osd.0 [v2:172.21.15.89:6802/2758701754,v1:172.21.15.89:6803/2758701754] boot 2024-09-21T04:04:21.265 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:04:21 smithi089 ceph-mon[23733]: osdmap e9: 1 total, 1 up, 1 in 2024-09-21T04:04:21.265 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:04:21 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-21T04:04:21.326 INFO:teuthology.orchestra.run.smithi089.stdout:Created osd(s) 0 on host 'smithi089' 2024-09-21T04:04:21.358 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:04:21 smithi195 ceph-mon[25840]: purged_snaps scrub starts 2024-09-21T04:04:21.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:04:21 smithi195 ceph-mon[25840]: purged_snaps scrub ok 2024-09-21T04:04:21.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:04:21 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-21T04:04:21.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:04:21 smithi195 ceph-mon[25840]: osd.0 [v2:172.21.15.89:6802/2758701754,v1:172.21.15.89:6803/2758701754] boot 2024-09-21T04:04:21.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:04:21 smithi195 ceph-mon[25840]: osdmap e9: 1 total, 1 up, 1 in 2024-09-21T04:04:21.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:04:21 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-21T04:04:22.264 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:04:22 smithi089 ceph-mon[23733]: pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T04:04:22.265 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:04:22 smithi089 ceph-mon[23733]: osdmap e10: 1 total, 1 up, 1 in 2024-09-21T04:04:22.265 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:04:22 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' 2024-09-21T04:04:22.358 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:04:22 smithi195 ceph-mon[25840]: pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T04:04:22.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:04:22 smithi195 ceph-mon[25840]: osdmap e10: 1 total, 1 up, 1 in 2024-09-21T04:04:22.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:04:22 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' 2024-09-21T04:04:23.014 DEBUG:teuthology.orchestra.run.smithi089:osd.0> sudo journalctl -f -n 0 -u ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490@osd.0.service 2024-09-21T04:04:23.016 INFO:tasks.cephadm:Deploying osd.1 on smithi089 with /dev/vg_nvme/lv_3... 2024-09-21T04:04:23.016 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 1b9b79fc-77ce-11ef-baf5-efdc52797490 -- lvm zap /dev/vg_nvme/lv_3 2024-09-21T04:04:23.765 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:04:23 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' 2024-09-21T04:04:23.765 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:04:23 smithi089 ceph-mon[23733]: Reconfiguring grafana.smithi089 (dependencies changed)... 2024-09-21T04:04:23.765 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:04:23 smithi089 ceph-mon[23733]: Reconfiguring daemon grafana.smithi089 on smithi089 2024-09-21T04:04:23.859 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:04:23 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' 2024-09-21T04:04:23.859 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:04:23 smithi195 ceph-mon[25840]: Reconfiguring grafana.smithi089 (dependencies changed)... 2024-09-21T04:04:23.859 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:04:23 smithi195 ceph-mon[25840]: Reconfiguring daemon grafana.smithi089 on smithi089 2024-09-21T04:04:24.858 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:04:24 smithi195 ceph-mon[25840]: pgmap v33: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-21T04:04:24.939 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:04:24 smithi089 ceph-mon[23733]: pgmap v33: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-21T04:04:26.858 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:04:26 smithi195 ceph-mon[25840]: pgmap v34: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-21T04:04:26.906 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:04:26 smithi089 ceph-mon[23733]: pgmap v34: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-21T04:04:28.951 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:04:28 smithi089 ceph-mon[23733]: pgmap v35: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-21T04:04:28.951 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:04:28 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' 2024-09-21T04:04:28.951 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:04:28 smithi089 ceph-mon[23733]: Reconfiguring crash.smithi195 (monmap changed)... 2024-09-21T04:04:28.951 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:04:28 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi195", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-09-21T04:04:28.951 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:04:28 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T04:04:28.951 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:04:28 smithi089 ceph-mon[23733]: Reconfiguring daemon crash.smithi195 on smithi195 2024-09-21T04:04:29.074 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:04:28 smithi195 ceph-mon[25840]: pgmap v35: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-21T04:04:29.075 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:04:28 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' 2024-09-21T04:04:29.075 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:04:28 smithi195 ceph-mon[25840]: Reconfiguring crash.smithi195 (monmap changed)... 2024-09-21T04:04:29.075 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:04:28 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi195", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-09-21T04:04:29.075 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:04:28 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T04:04:29.075 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:04:28 smithi195 ceph-mon[25840]: Reconfiguring daemon crash.smithi195 on smithi195 2024-09-21T04:04:29.082 INFO:teuthology.orchestra.run.smithi089.stdout: 2024-09-21T04:04:29.102 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 1b9b79fc-77ce-11ef-baf5-efdc52797490 -- ceph orch daemon add osd smithi089:vg_nvme/lv_3 2024-09-21T04:04:30.314 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:04:30 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' 2024-09-21T04:04:30.314 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:04:30 smithi195 ceph-mon[25840]: Reconfiguring mgr.smithi195.cgotzm (monmap changed)... 2024-09-21T04:04:30.314 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:04:30 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi195.cgotzm", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-21T04:04:30.314 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:04:30 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-21T04:04:30.314 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:04:30 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T04:04:30.314 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:04:30 smithi195 ceph-mon[25840]: Reconfiguring daemon mgr.smithi195.cgotzm on smithi195 2024-09-21T04:04:30.314 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:04:30 smithi195 ceph-mon[25840]: pgmap v36: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-21T04:04:30.317 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:04:30 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' 2024-09-21T04:04:30.317 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:04:30 smithi089 ceph-mon[23733]: Reconfiguring mgr.smithi195.cgotzm (monmap changed)... 2024-09-21T04:04:30.317 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:04:30 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi195.cgotzm", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-21T04:04:30.317 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:04:30 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-21T04:04:30.317 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:04:30 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T04:04:30.317 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:04:30 smithi089 ceph-mon[23733]: Reconfiguring daemon mgr.smithi195.cgotzm on smithi195 2024-09-21T04:04:30.318 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:04:30 smithi089 ceph-mon[23733]: pgmap v36: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-21T04:04:31.543 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:04:31 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' 2024-09-21T04:04:31.543 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:04:31 smithi195 ceph-mon[25840]: Reconfiguring mon.smithi195 (monmap changed)... 2024-09-21T04:04:31.543 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:04:31 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-21T04:04:31.543 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:04:31 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-21T04:04:31.543 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:04:31 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T04:04:31.543 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:04:31 smithi195 ceph-mon[25840]: Reconfiguring daemon mon.smithi195 on smithi195 2024-09-21T04:04:31.543 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:04:31 smithi195 ceph-mon[25840]: from='client.14252 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi089:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T04:04:31.543 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:04:31 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-21T04:04:31.543 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:04:31 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-21T04:04:31.543 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:04:31 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T04:04:31.590 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:04:31 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' 2024-09-21T04:04:31.590 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:04:31 smithi089 ceph-mon[23733]: Reconfiguring mon.smithi195 (monmap changed)... 2024-09-21T04:04:31.590 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:04:31 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-21T04:04:31.590 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:04:31 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-21T04:04:31.590 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:04:31 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T04:04:31.590 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:04:31 smithi089 ceph-mon[23733]: Reconfiguring daemon mon.smithi195 on smithi195 2024-09-21T04:04:31.590 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:04:31 smithi089 ceph-mon[23733]: from='client.14252 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi089:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T04:04:31.591 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:04:31 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-21T04:04:31.591 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:04:31 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-21T04:04:31.591 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:04:31 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T04:04:32.766 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:04:32 smithi089 ceph-mon[23733]: pgmap v37: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-21T04:04:32.766 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:04:32 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' 2024-09-21T04:04:32.766 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:04:32 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-21T04:04:32.766 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:04:32 smithi089 ceph-mon[23733]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-21T04:04:32.858 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:04:32 smithi195 ceph-mon[25840]: pgmap v37: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-21T04:04:32.859 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:04:32 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' 2024-09-21T04:04:32.859 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:04:32 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-21T04:04:32.859 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:04:32 smithi195 ceph-mon[25840]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-21T04:04:33.859 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:04:33 smithi195 ceph-mon[25840]: from='client.? 172.21.15.89:0/1018179361' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "3e63078d-befa-41e7-b13f-11fd3a63588a"}]: dispatch 2024-09-21T04:04:33.859 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:04:33 smithi195 ceph-mon[25840]: from='client.? 172.21.15.89:0/1018179361' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "3e63078d-befa-41e7-b13f-11fd3a63588a"}]': finished 2024-09-21T04:04:33.859 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:04:33 smithi195 ceph-mon[25840]: osdmap e11: 2 total, 1 up, 2 in 2024-09-21T04:04:33.859 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:04:33 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-21T04:04:34.015 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:04:33 smithi089 ceph-mon[23733]: from='client.? 172.21.15.89:0/1018179361' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "3e63078d-befa-41e7-b13f-11fd3a63588a"}]: dispatch 2024-09-21T04:04:34.015 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:04:33 smithi089 ceph-mon[23733]: from='client.? 172.21.15.89:0/1018179361' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "3e63078d-befa-41e7-b13f-11fd3a63588a"}]': finished 2024-09-21T04:04:34.015 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:04:33 smithi089 ceph-mon[23733]: osdmap e11: 2 total, 1 up, 2 in 2024-09-21T04:04:34.015 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:04:33 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-21T04:04:34.858 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:04:34 smithi195 ceph-mon[25840]: pgmap v38: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-21T04:04:34.859 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:04:34 smithi195 ceph-mon[25840]: from='client.? 172.21.15.89:0/882064647' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-21T04:04:35.014 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:04:34 smithi089 ceph-mon[23733]: pgmap v38: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-21T04:04:35.014 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:04:34 smithi089 ceph-mon[23733]: from='client.? 172.21.15.89:0/882064647' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-21T04:04:36.858 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:04:36 smithi195 ceph-mon[25840]: pgmap v40: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-21T04:04:36.965 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:04:36 smithi089 ceph-mon[23733]: pgmap v40: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-21T04:04:38.858 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:04:38 smithi195 ceph-mon[25840]: pgmap v41: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-21T04:04:38.891 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:04:38 smithi089 ceph-mon[23733]: pgmap v41: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-21T04:04:39.858 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:04:39 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-09-21T04:04:39.859 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:04:39 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T04:04:39.873 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:04:39 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-09-21T04:04:39.873 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:04:39 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T04:04:40.855 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:04:40 smithi089 ceph-mon[23733]: pgmap v42: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-21T04:04:40.855 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:04:40 smithi089 ceph-mon[23733]: Deploying daemon osd.1 on smithi089 2024-09-21T04:04:40.858 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:04:40 smithi195 ceph-mon[25840]: pgmap v42: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-21T04:04:40.858 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:04:40 smithi195 ceph-mon[25840]: Deploying daemon osd.1 on smithi089 2024-09-21T04:04:42.858 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:04:42 smithi195 ceph-mon[25840]: pgmap v43: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-21T04:04:43.015 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:04:42 smithi089 ceph-mon[23733]: pgmap v43: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-21T04:04:44.764 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:04:44 smithi089 ceph-mon[23733]: pgmap v44: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-21T04:04:44.764 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:04:44 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' 2024-09-21T04:04:44.858 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:04:44 smithi195 ceph-mon[25840]: pgmap v44: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-21T04:04:44.859 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:04:44 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' 2024-09-21T04:04:46.752 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:04:46 smithi089 ceph-mon[23733]: pgmap v45: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-21T04:04:46.858 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:04:46 smithi195 ceph-mon[25840]: pgmap v45: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-21T04:04:47.092 INFO:teuthology.orchestra.run.smithi089.stdout:Created osd(s) 1 on host 'smithi089' 2024-09-21T04:04:47.695 DEBUG:teuthology.orchestra.run.smithi089:osd.1> sudo journalctl -f -n 0 -u ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490@osd.1.service 2024-09-21T04:04:47.697 INFO:tasks.cephadm:Deploying osd.2 on smithi089 with /dev/vg_nvme/lv_2... 2024-09-21T04:04:47.698 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 1b9b79fc-77ce-11ef-baf5-efdc52797490 -- lvm zap /dev/vg_nvme/lv_2 2024-09-21T04:04:48.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:04:48 smithi195 ceph-mon[25840]: pgmap v46: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-21T04:04:48.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:04:48 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' 2024-09-21T04:04:48.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:04:48 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.89:9093"}]: dispatch 2024-09-21T04:04:48.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:04:48 smithi195 ceph-mon[25840]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.89:9093"}]: dispatch 2024-09-21T04:04:48.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:04:48 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' 2024-09-21T04:04:48.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:04:48 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-21T04:04:48.360 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:04:48 smithi195 ceph-mon[25840]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-21T04:04:48.360 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:04:48 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.89:3000"}]: dispatch 2024-09-21T04:04:48.360 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:04:48 smithi195 ceph-mon[25840]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.89:3000"}]: dispatch 2024-09-21T04:04:48.360 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:04:48 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' 2024-09-21T04:04:48.360 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:04:48 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-21T04:04:48.360 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:04:48 smithi195 ceph-mon[25840]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-21T04:04:48.360 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:04:48 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.89:9095"}]: dispatch 2024-09-21T04:04:48.360 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:04:48 smithi195 ceph-mon[25840]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.89:9095"}]: dispatch 2024-09-21T04:04:48.360 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:04:48 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' 2024-09-21T04:04:48.360 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:04:48 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T04:04:48.360 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:04:48 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T04:04:48.360 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:04:48 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T04:04:48.360 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:04:48 smithi195 ceph-mon[25840]: from='osd.1 [v2:172.21.15.89:6810/2620086826,v1:172.21.15.89:6811/2620086826]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-09-21T04:04:48.515 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:04:48 smithi089 ceph-mon[23733]: pgmap v46: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-21T04:04:48.515 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:04:48 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' 2024-09-21T04:04:48.515 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:04:48 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.89:9093"}]: dispatch 2024-09-21T04:04:48.515 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:04:48 smithi089 ceph-mon[23733]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.89:9093"}]: dispatch 2024-09-21T04:04:48.515 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:04:48 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' 2024-09-21T04:04:48.515 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:04:48 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-21T04:04:48.515 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:04:48 smithi089 ceph-mon[23733]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-21T04:04:48.515 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:04:48 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.89:3000"}]: dispatch 2024-09-21T04:04:48.515 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:04:48 smithi089 ceph-mon[23733]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.89:3000"}]: dispatch 2024-09-21T04:04:48.515 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:04:48 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' 2024-09-21T04:04:48.515 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:04:48 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-21T04:04:48.516 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:04:48 smithi089 ceph-mon[23733]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-21T04:04:48.516 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:04:48 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.89:9095"}]: dispatch 2024-09-21T04:04:48.516 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:04:48 smithi089 ceph-mon[23733]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.89:9095"}]: dispatch 2024-09-21T04:04:48.516 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:04:48 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' 2024-09-21T04:04:48.516 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:04:48 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T04:04:48.516 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:04:48 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T04:04:48.516 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:04:48 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T04:04:48.516 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:04:48 smithi089 ceph-mon[23733]: from='osd.1 [v2:172.21.15.89:6810/2620086826,v1:172.21.15.89:6811/2620086826]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-09-21T04:04:49.265 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:04:49 smithi089 ceph-mon[23733]: from='osd.1 [v2:172.21.15.89:6810/2620086826,v1:172.21.15.89:6811/2620086826]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-09-21T04:04:49.265 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:04:49 smithi089 ceph-mon[23733]: osdmap e12: 2 total, 1 up, 2 in 2024-09-21T04:04:49.265 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:04:49 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-21T04:04:49.265 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:04:49 smithi089 ceph-mon[23733]: from='osd.1 [v2:172.21.15.89:6810/2620086826,v1:172.21.15.89:6811/2620086826]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi089", "root=default"]}]: dispatch 2024-09-21T04:04:49.265 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:04:49 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' 2024-09-21T04:04:49.265 INFO:journalctl@ceph.osd.1.smithi089.stdout:Sep 21 04:04:49 smithi089 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-1[37020]: 2024-09-21T04:04:49.134+0000 7f97e6f29700 -1 osd.1 0 waiting for initial osdmap 2024-09-21T04:04:49.265 INFO:journalctl@ceph.osd.1.smithi089.stdout:Sep 21 04:04:49 smithi089 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-1[37020]: 2024-09-21T04:04:49.137+0000 7f97e18c0700 -1 osd.1 13 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-21T04:04:49.358 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:04:49 smithi195 ceph-mon[25840]: from='osd.1 [v2:172.21.15.89:6810/2620086826,v1:172.21.15.89:6811/2620086826]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-09-21T04:04:49.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:04:49 smithi195 ceph-mon[25840]: osdmap e12: 2 total, 1 up, 2 in 2024-09-21T04:04:49.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:04:49 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-21T04:04:49.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:04:49 smithi195 ceph-mon[25840]: from='osd.1 [v2:172.21.15.89:6810/2620086826,v1:172.21.15.89:6811/2620086826]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi089", "root=default"]}]: dispatch 2024-09-21T04:04:49.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:04:49 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' 2024-09-21T04:04:50.212 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:04:50 smithi089 ceph-mon[23733]: pgmap v48: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-21T04:04:50.212 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:04:50 smithi089 ceph-mon[23733]: from='osd.1 [v2:172.21.15.89:6810/2620086826,v1:172.21.15.89:6811/2620086826]' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi089", "root=default"]}]': finished 2024-09-21T04:04:50.212 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:04:50 smithi089 ceph-mon[23733]: osdmap e13: 2 total, 1 up, 2 in 2024-09-21T04:04:50.212 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:04:50 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-21T04:04:50.213 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:04:50 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-21T04:04:50.595 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:04:50 smithi195 ceph-mon[25840]: pgmap v48: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-21T04:04:50.595 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:04:50 smithi195 ceph-mon[25840]: from='osd.1 [v2:172.21.15.89:6810/2620086826,v1:172.21.15.89:6811/2620086826]' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi089", "root=default"]}]': finished 2024-09-21T04:04:50.596 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:04:50 smithi195 ceph-mon[25840]: osdmap e13: 2 total, 1 up, 2 in 2024-09-21T04:04:50.596 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:04:50 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-21T04:04:50.596 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:04:50 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-21T04:04:51.514 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:04:51 smithi089 ceph-mon[23733]: purged_snaps scrub starts 2024-09-21T04:04:51.515 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:04:51 smithi089 ceph-mon[23733]: purged_snaps scrub ok 2024-09-21T04:04:51.515 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:04:51 smithi089 ceph-mon[23733]: osd.1 [v2:172.21.15.89:6810/2620086826,v1:172.21.15.89:6811/2620086826] boot 2024-09-21T04:04:51.515 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:04:51 smithi089 ceph-mon[23733]: osdmap e14: 2 total, 2 up, 2 in 2024-09-21T04:04:51.515 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:04:51 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-21T04:04:51.608 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:04:51 smithi195 ceph-mon[25840]: purged_snaps scrub starts 2024-09-21T04:04:51.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:04:51 smithi195 ceph-mon[25840]: purged_snaps scrub ok 2024-09-21T04:04:51.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:04:51 smithi195 ceph-mon[25840]: osd.1 [v2:172.21.15.89:6810/2620086826,v1:172.21.15.89:6811/2620086826] boot 2024-09-21T04:04:51.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:04:51 smithi195 ceph-mon[25840]: osdmap e14: 2 total, 2 up, 2 in 2024-09-21T04:04:51.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:04:51 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-21T04:04:51.860 INFO:teuthology.orchestra.run.smithi089.stdout: 2024-09-21T04:04:51.879 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 1b9b79fc-77ce-11ef-baf5-efdc52797490 -- ceph orch daemon add osd smithi089:vg_nvme/lv_2 2024-09-21T04:04:52.481 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:04:52 smithi089 ceph-mon[23733]: pgmap v51: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-21T04:04:52.482 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:04:52 smithi089 ceph-mon[23733]: osdmap e15: 2 total, 2 up, 2 in 2024-09-21T04:04:52.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:04:52 smithi195 ceph-mon[25840]: pgmap v51: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-21T04:04:52.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:04:52 smithi195 ceph-mon[25840]: osdmap e15: 2 total, 2 up, 2 in 2024-09-21T04:04:54.477 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:04:54 smithi089 ceph-mon[23733]: pgmap v53: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-21T04:04:54.608 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:04:54 smithi195 ceph-mon[25840]: pgmap v53: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-21T04:04:56.264 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:04:56 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' 2024-09-21T04:04:56.265 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:04:56 smithi089 ceph-mon[23733]: pgmap v54: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-21T04:04:56.265 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:04:56 smithi089 ceph-mon[23733]: from='client.14270 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi089:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T04:04:56.265 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:04:56 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-21T04:04:56.265 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:04:56 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-21T04:04:56.265 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:04:56 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T04:04:56.358 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:04:56 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' 2024-09-21T04:04:56.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:04:56 smithi195 ceph-mon[25840]: pgmap v54: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-21T04:04:56.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:04:56 smithi195 ceph-mon[25840]: from='client.14270 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi089:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T04:04:56.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:04:56 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-21T04:04:56.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:04:56 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-21T04:04:56.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:04:56 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T04:04:58.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:04:58 smithi195 ceph-mon[25840]: pgmap v55: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-21T04:04:58.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:04:58 smithi195 ceph-mon[25840]: from='client.? 172.21.15.89:0/869656407' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "99ec6c1f-c76e-4f79-b030-b980e297e1cb"}]: dispatch 2024-09-21T04:04:58.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:04:58 smithi195 ceph-mon[25840]: from='client.? 172.21.15.89:0/869656407' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "99ec6c1f-c76e-4f79-b030-b980e297e1cb"}]': finished 2024-09-21T04:04:58.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:04:58 smithi195 ceph-mon[25840]: osdmap e16: 3 total, 2 up, 3 in 2024-09-21T04:04:58.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:04:58 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-21T04:04:58.515 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:04:58 smithi089 ceph-mon[23733]: pgmap v55: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-21T04:04:58.515 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:04:58 smithi089 ceph-mon[23733]: from='client.? 172.21.15.89:0/869656407' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "99ec6c1f-c76e-4f79-b030-b980e297e1cb"}]: dispatch 2024-09-21T04:04:58.515 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:04:58 smithi089 ceph-mon[23733]: from='client.? 172.21.15.89:0/869656407' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "99ec6c1f-c76e-4f79-b030-b980e297e1cb"}]': finished 2024-09-21T04:04:58.515 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:04:58 smithi089 ceph-mon[23733]: osdmap e16: 3 total, 2 up, 3 in 2024-09-21T04:04:58.515 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:04:58 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-21T04:04:59.358 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:04:59 smithi195 ceph-mon[25840]: from='client.? 172.21.15.89:0/2901884668' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-21T04:04:59.514 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:04:59 smithi089 ceph-mon[23733]: from='client.? 172.21.15.89:0/2901884668' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-21T04:05:00.358 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:05:00 smithi195 ceph-mon[25840]: pgmap v57: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-21T04:05:00.425 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:05:00 smithi089 ceph-mon[23733]: pgmap v57: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-21T04:05:02.264 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:05:02 smithi089 ceph-mon[23733]: pgmap v58: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-21T04:05:02.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:05:02 smithi195 ceph-mon[25840]: pgmap v58: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-21T04:05:04.108 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:05:03 smithi195 ceph-mon[25840]: Detected new or changed devices on smithi089 2024-09-21T04:05:04.109 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:05:03 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' 2024-09-21T04:05:04.109 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:05:03 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-21T04:05:04.109 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:05:03 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-21T04:05:04.109 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:05:03 smithi195 ceph-mon[25840]: Adjusting osd_memory_target on smithi089 to 6431M 2024-09-21T04:05:04.109 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:05:03 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' 2024-09-21T04:05:04.109 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:05:03 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' 2024-09-21T04:05:04.109 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:05:03 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T04:05:04.109 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:05:03 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T04:05:04.109 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:05:03 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T04:05:04.265 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:05:03 smithi089 ceph-mon[23733]: Detected new or changed devices on smithi089 2024-09-21T04:05:04.265 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:05:03 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' 2024-09-21T04:05:04.265 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:05:03 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-21T04:05:04.265 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:05:03 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-21T04:05:04.265 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:05:03 smithi089 ceph-mon[23733]: Adjusting osd_memory_target on smithi089 to 6431M 2024-09-21T04:05:04.265 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:05:03 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' 2024-09-21T04:05:04.265 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:05:03 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' 2024-09-21T04:05:04.265 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:05:03 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T04:05:04.265 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:05:03 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T04:05:04.265 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:05:03 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T04:05:05.014 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:05:04 smithi089 ceph-mon[23733]: pgmap v59: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-21T04:05:05.014 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:05:04 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' 2024-09-21T04:05:05.015 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:05:04 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-09-21T04:05:05.015 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:05:04 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T04:05:05.280 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:05:04 smithi195 ceph-mon[25840]: pgmap v59: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-21T04:05:05.281 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:05:04 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' 2024-09-21T04:05:05.281 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:05:04 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-09-21T04:05:05.281 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:05:04 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T04:05:06.014 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:05:05 smithi089 ceph-mon[23733]: Deploying daemon osd.2 on smithi089 2024-09-21T04:05:06.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:05:05 smithi195 ceph-mon[25840]: Deploying daemon osd.2 on smithi089 2024-09-21T04:05:07.123 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:05:06 smithi089 ceph-mon[23733]: pgmap v60: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-21T04:05:07.358 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:05:06 smithi195 ceph-mon[25840]: pgmap v60: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-21T04:05:09.015 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:05:08 smithi089 ceph-mon[23733]: pgmap v61: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-21T04:05:09.358 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:05:08 smithi195 ceph-mon[25840]: pgmap v61: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-21T04:05:11.264 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:05:10 smithi089 ceph-mon[23733]: pgmap v62: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-21T04:05:11.265 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:05:10 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' 2024-09-21T04:05:11.358 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:05:10 smithi195 ceph-mon[25840]: pgmap v62: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-21T04:05:11.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:05:10 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' 2024-09-21T04:05:13.097 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:05:12 smithi089 ceph-mon[23733]: pgmap v63: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-21T04:05:13.098 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:05:12 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' 2024-09-21T04:05:13.358 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:05:12 smithi195 ceph-mon[25840]: pgmap v63: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-21T04:05:13.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:05:12 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' 2024-09-21T04:05:14.223 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:05:13 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi089.srlyxf/mirror_snapshot_schedule"}]: dispatch 2024-09-21T04:05:14.223 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:05:13 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi089.srlyxf/trash_purge_schedule"}]: dispatch 2024-09-21T04:05:14.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:05:13 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi089.srlyxf/mirror_snapshot_schedule"}]: dispatch 2024-09-21T04:05:14.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:05:13 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi089.srlyxf/trash_purge_schedule"}]: dispatch 2024-09-21T04:05:15.014 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:05:14 smithi089 ceph-mon[23733]: pgmap v64: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-21T04:05:15.015 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:05:14 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' 2024-09-21T04:05:15.015 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:05:14 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' 2024-09-21T04:05:15.015 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:05:14 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T04:05:15.015 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:05:14 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T04:05:15.015 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:05:14 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T04:05:15.152 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:05:14 smithi195 ceph-mon[25840]: pgmap v64: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-21T04:05:15.152 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:05:14 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' 2024-09-21T04:05:15.152 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:05:14 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' 2024-09-21T04:05:15.152 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:05:14 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T04:05:15.152 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:05:14 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T04:05:15.152 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:05:14 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T04:05:16.014 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:05:15 smithi089 ceph-mon[23733]: from='osd.2 [v2:172.21.15.89:6818/3898997901,v1:172.21.15.89:6819/3898997901]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-09-21T04:05:16.109 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:05:15 smithi195 ceph-mon[25840]: from='osd.2 [v2:172.21.15.89:6818/3898997901,v1:172.21.15.89:6819/3898997901]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-09-21T04:05:16.841 INFO:teuthology.orchestra.run.smithi089.stdout:Created osd(s) 2 on host 'smithi089' 2024-09-21T04:05:17.215 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:05:16 smithi089 ceph-mon[23733]: pgmap v65: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-21T04:05:17.215 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:05:16 smithi089 ceph-mon[23733]: from='osd.2 [v2:172.21.15.89:6818/3898997901,v1:172.21.15.89:6819/3898997901]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-09-21T04:05:17.215 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:05:16 smithi089 ceph-mon[23733]: osdmap e17: 3 total, 2 up, 3 in 2024-09-21T04:05:17.215 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:05:16 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-21T04:05:17.215 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:05:16 smithi089 ceph-mon[23733]: from='osd.2 [v2:172.21.15.89:6818/3898997901,v1:172.21.15.89:6819/3898997901]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi089", "root=default"]}]: dispatch 2024-09-21T04:05:17.215 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:05:16 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' 2024-09-21T04:05:17.215 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:05:16 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' 2024-09-21T04:05:17.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:05:16 smithi195 ceph-mon[25840]: pgmap v65: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-21T04:05:17.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:05:16 smithi195 ceph-mon[25840]: from='osd.2 [v2:172.21.15.89:6818/3898997901,v1:172.21.15.89:6819/3898997901]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-09-21T04:05:17.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:05:16 smithi195 ceph-mon[25840]: osdmap e17: 3 total, 2 up, 3 in 2024-09-21T04:05:17.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:05:16 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-21T04:05:17.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:05:16 smithi195 ceph-mon[25840]: from='osd.2 [v2:172.21.15.89:6818/3898997901,v1:172.21.15.89:6819/3898997901]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi089", "root=default"]}]: dispatch 2024-09-21T04:05:17.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:05:16 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' 2024-09-21T04:05:17.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:05:16 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' 2024-09-21T04:05:17.490 DEBUG:teuthology.orchestra.run.smithi089:osd.2> sudo journalctl -f -n 0 -u ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490@osd.2.service 2024-09-21T04:05:17.492 INFO:tasks.cephadm:Deploying osd.3 on smithi089 with /dev/vg_nvme/lv_1... 2024-09-21T04:05:17.492 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 1b9b79fc-77ce-11ef-baf5-efdc52797490 -- lvm zap /dev/vg_nvme/lv_1 2024-09-21T04:05:18.095 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:05:17 smithi089 ceph-mon[23733]: from='osd.2 [v2:172.21.15.89:6818/3898997901,v1:172.21.15.89:6819/3898997901]' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi089", "root=default"]}]': finished 2024-09-21T04:05:18.096 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:05:17 smithi089 ceph-mon[23733]: osdmap e18: 3 total, 2 up, 3 in 2024-09-21T04:05:18.096 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:05:17 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-21T04:05:18.096 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:05:17 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-21T04:05:18.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:05:17 smithi195 ceph-mon[25840]: from='osd.2 [v2:172.21.15.89:6818/3898997901,v1:172.21.15.89:6819/3898997901]' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi089", "root=default"]}]': finished 2024-09-21T04:05:18.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:05:17 smithi195 ceph-mon[25840]: osdmap e18: 3 total, 2 up, 3 in 2024-09-21T04:05:18.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:05:17 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-21T04:05:18.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:05:17 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-21T04:05:19.123 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:05:18 smithi089 ceph-mon[23733]: purged_snaps scrub starts 2024-09-21T04:05:19.123 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:05:18 smithi089 ceph-mon[23733]: purged_snaps scrub ok 2024-09-21T04:05:19.123 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:05:18 smithi089 ceph-mon[23733]: pgmap v68: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-21T04:05:19.123 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:05:18 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-21T04:05:19.123 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:05:18 smithi089 ceph-mon[23733]: osd.2 [v2:172.21.15.89:6818/3898997901,v1:172.21.15.89:6819/3898997901] boot 2024-09-21T04:05:19.124 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:05:18 smithi089 ceph-mon[23733]: osdmap e19: 3 total, 3 up, 3 in 2024-09-21T04:05:19.124 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:05:18 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-21T04:05:19.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:05:18 smithi195 ceph-mon[25840]: purged_snaps scrub starts 2024-09-21T04:05:19.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:05:18 smithi195 ceph-mon[25840]: purged_snaps scrub ok 2024-09-21T04:05:19.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:05:18 smithi195 ceph-mon[25840]: pgmap v68: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-21T04:05:19.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:05:18 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-21T04:05:19.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:05:18 smithi195 ceph-mon[25840]: osd.2 [v2:172.21.15.89:6818/3898997901,v1:172.21.15.89:6819/3898997901] boot 2024-09-21T04:05:19.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:05:18 smithi195 ceph-mon[25840]: osdmap e19: 3 total, 3 up, 3 in 2024-09-21T04:05:19.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:05:18 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-21T04:05:20.264 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:05:19 smithi089 ceph-mon[23733]: osdmap e20: 3 total, 3 up, 3 in 2024-09-21T04:05:20.264 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:05:19 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2024-09-21T04:05:20.311 INFO:teuthology.orchestra.run.smithi089.stdout: 2024-09-21T04:05:20.330 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 1b9b79fc-77ce-11ef-baf5-efdc52797490 -- ceph orch daemon add osd smithi089:vg_nvme/lv_1 2024-09-21T04:05:20.358 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:05:19 smithi195 ceph-mon[25840]: osdmap e20: 3 total, 3 up, 3 in 2024-09-21T04:05:20.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:05:19 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2024-09-21T04:05:21.003 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:05:20 smithi089 ceph-mon[23733]: pgmap v71: 0 pgs: ; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-09-21T04:05:21.003 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:05:20 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2024-09-21T04:05:21.003 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:05:20 smithi089 ceph-mon[23733]: osdmap e21: 3 total, 3 up, 3 in 2024-09-21T04:05:21.003 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:05:20 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-09-21T04:05:21.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:05:20 smithi195 ceph-mon[25840]: pgmap v71: 0 pgs: ; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-09-21T04:05:21.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:05:20 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2024-09-21T04:05:21.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:05:20 smithi195 ceph-mon[25840]: osdmap e21: 3 total, 3 up, 3 in 2024-09-21T04:05:21.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:05:20 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-09-21T04:05:22.264 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:05:21 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2024-09-21T04:05:22.264 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:05:21 smithi089 ceph-mon[23733]: osdmap e22: 3 total, 3 up, 3 in 2024-09-21T04:05:22.265 INFO:journalctl@ceph.osd.0.smithi089.stdout:Sep 21 04:05:22 smithi089 sudo[42243]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-09-21T04:05:22.265 INFO:journalctl@ceph.osd.0.smithi089.stdout:Sep 21 04:05:22 smithi089 sudo[42243]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-21T04:05:22.265 INFO:journalctl@ceph.osd.0.smithi089.stdout:Sep 21 04:05:22 smithi089 sudo[42243]: pam_unix(sudo:session): session closed for user root 2024-09-21T04:05:22.358 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:05:21 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2024-09-21T04:05:22.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:05:21 smithi195 ceph-mon[25840]: osdmap e22: 3 total, 3 up, 3 in 2024-09-21T04:05:22.649 INFO:journalctl@ceph.osd.0.smithi089.stdout:Sep 21 04:05:22 smithi089 sudo[42248]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-09-21T04:05:22.649 INFO:journalctl@ceph.osd.0.smithi089.stdout:Sep 21 04:05:22 smithi089 sudo[42248]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-21T04:05:22.649 INFO:journalctl@ceph.osd.0.smithi089.stdout:Sep 21 04:05:22 smithi089 sudo[42248]: pam_unix(sudo:session): session closed for user root 2024-09-21T04:05:22.911 INFO:journalctl@ceph.osd.1.smithi089.stdout:Sep 21 04:05:22 smithi089 sudo[42251]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-09-21T04:05:22.911 INFO:journalctl@ceph.osd.1.smithi089.stdout:Sep 21 04:05:22 smithi089 sudo[42251]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-21T04:05:22.912 INFO:journalctl@ceph.osd.1.smithi089.stdout:Sep 21 04:05:22 smithi089 sudo[42251]: pam_unix(sudo:session): session closed for user root 2024-09-21T04:05:23.168 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:05:22 smithi089 ceph-mon[23733]: pgmap v74: 1 pgs: 1 unknown; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-09-21T04:05:23.168 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:05:22 smithi089 ceph-mon[23733]: osdmap e23: 3 total, 3 up, 3 in 2024-09-21T04:05:23.168 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:05:22 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' 2024-09-21T04:05:23.168 INFO:journalctl@ceph.osd.1.smithi089.stdout:Sep 21 04:05:22 smithi089 sudo[42272]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-09-21T04:05:23.168 INFO:journalctl@ceph.osd.1.smithi089.stdout:Sep 21 04:05:22 smithi089 sudo[42272]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-21T04:05:23.169 INFO:journalctl@ceph.osd.1.smithi089.stdout:Sep 21 04:05:22 smithi089 sudo[42272]: pam_unix(sudo:session): session closed for user root 2024-09-21T04:05:23.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:05:22 smithi195 ceph-mon[25840]: pgmap v74: 1 pgs: 1 unknown; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-09-21T04:05:23.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:05:22 smithi195 ceph-mon[25840]: osdmap e23: 3 total, 3 up, 3 in 2024-09-21T04:05:23.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:05:22 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' 2024-09-21T04:05:23.435 INFO:journalctl@ceph.osd.2.smithi089.stdout:Sep 21 04:05:23 smithi089 sudo[42303]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-09-21T04:05:23.435 INFO:journalctl@ceph.osd.2.smithi089.stdout:Sep 21 04:05:23 smithi089 sudo[42303]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-21T04:05:23.435 INFO:journalctl@ceph.osd.2.smithi089.stdout:Sep 21 04:05:23 smithi089 sudo[42303]: pam_unix(sudo:session): session closed for user root 2024-09-21T04:05:23.687 INFO:journalctl@ceph.osd.2.smithi089.stdout:Sep 21 04:05:23 smithi089 sudo[42370]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-09-21T04:05:23.687 INFO:journalctl@ceph.osd.2.smithi089.stdout:Sep 21 04:05:23 smithi089 sudo[42370]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-21T04:05:23.687 INFO:journalctl@ceph.osd.2.smithi089.stdout:Sep 21 04:05:23 smithi089 sudo[42370]: pam_unix(sudo:session): session closed for user root 2024-09-21T04:05:23.945 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:05:23 smithi089 sudo[42385]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-09-21T04:05:23.945 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:05:23 smithi089 sudo[42385]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-21T04:05:23.945 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:05:23 smithi089 sudo[42385]: pam_unix(sudo:session): session closed for user root 2024-09-21T04:05:24.212 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:05:23 smithi089 sudo[42468]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-09-21T04:05:24.212 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:05:23 smithi089 sudo[42468]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-21T04:05:24.212 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:05:23 smithi089 sudo[42468]: pam_unix(sudo:session): session closed for user root 2024-09-21T04:05:24.212 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:05:23 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-21T04:05:24.212 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:05:23 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-21T04:05:24.212 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:05:23 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T04:05:24.221 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:05:23 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-21T04:05:24.221 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:05:23 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-21T04:05:24.221 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:05:23 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T04:05:24.221 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:05:24 smithi195 sudo[28134]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-09-21T04:05:24.221 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:05:24 smithi195 sudo[28134]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-21T04:05:24.221 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:05:24 smithi195 sudo[28134]: pam_unix(sudo:session): session closed for user root 2024-09-21T04:05:24.608 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:05:24 smithi195 sudo[28137]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-09-21T04:05:24.608 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:05:24 smithi195 sudo[28137]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-21T04:05:24.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:05:24 smithi195 sudo[28137]: pam_unix(sudo:session): session closed for user root 2024-09-21T04:05:25.043 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:05:24 smithi089 ceph-mon[23733]: pgmap v76: 1 pgs: 1 creating+activating; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-09-21T04:05:25.044 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:05:24 smithi089 ceph-mon[23733]: from='client.14292 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi089:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T04:05:25.044 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:05:24 smithi089 ceph-mon[23733]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-21T04:05:25.044 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:05:24 smithi089 ceph-mon[23733]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-21T04:05:25.044 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:05:24 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "mon metadata", "id": "smithi089"}]: dispatch 2024-09-21T04:05:25.044 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:05:24 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "mon metadata", "id": "smithi195"}]: dispatch 2024-09-21T04:05:25.044 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:05:24 smithi089 ceph-mon[23733]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-21T04:05:25.044 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:05:24 smithi089 ceph-mon[23733]: mgrmap e19: smithi089.srlyxf(active, since 2m), standbys: smithi195.cgotzm 2024-09-21T04:05:25.044 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:05:24 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "mon metadata", "id": "smithi089"}]: dispatch 2024-09-21T04:05:25.044 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:05:24 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "mon metadata", "id": "smithi195"}]: dispatch 2024-09-21T04:05:25.044 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:05:24 smithi089 ceph-mon[23733]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-21T04:05:25.044 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:05:24 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' 2024-09-21T04:05:25.044 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:05:24 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' 2024-09-21T04:05:25.358 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:05:24 smithi195 ceph-mon[25840]: pgmap v76: 1 pgs: 1 creating+activating; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-09-21T04:05:25.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:05:24 smithi195 ceph-mon[25840]: from='client.14292 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi089:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T04:05:25.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:05:24 smithi195 ceph-mon[25840]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-21T04:05:25.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:05:24 smithi195 ceph-mon[25840]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-21T04:05:25.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:05:24 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "mon metadata", "id": "smithi089"}]: dispatch 2024-09-21T04:05:25.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:05:24 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "mon metadata", "id": "smithi195"}]: dispatch 2024-09-21T04:05:25.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:05:24 smithi195 ceph-mon[25840]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-21T04:05:25.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:05:24 smithi195 ceph-mon[25840]: mgrmap e19: smithi089.srlyxf(active, since 2m), standbys: smithi195.cgotzm 2024-09-21T04:05:25.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:05:24 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "mon metadata", "id": "smithi089"}]: dispatch 2024-09-21T04:05:25.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:05:24 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "mon metadata", "id": "smithi195"}]: dispatch 2024-09-21T04:05:25.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:05:24 smithi195 ceph-mon[25840]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-21T04:05:25.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:05:24 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' 2024-09-21T04:05:25.360 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:05:24 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' 2024-09-21T04:05:27.264 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:05:26 smithi089 ceph-mon[23733]: pgmap v77: 1 pgs: 1 creating+activating; 0 B data, 16 MiB used, 268 GiB / 268 GiB avail 2024-09-21T04:05:27.264 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:05:26 smithi089 ceph-mon[23733]: from='client.? 172.21.15.89:0/1170865039' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "288ca940-ca66-41f1-894f-32ac0c9d9eb3"}]: dispatch 2024-09-21T04:05:27.264 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:05:26 smithi089 ceph-mon[23733]: from='client.? 172.21.15.89:0/1170865039' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "288ca940-ca66-41f1-894f-32ac0c9d9eb3"}]': finished 2024-09-21T04:05:27.265 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:05:26 smithi089 ceph-mon[23733]: osdmap e24: 4 total, 3 up, 4 in 2024-09-21T04:05:27.265 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:05:26 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-21T04:05:27.265 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:05:26 smithi089 ceph-mon[23733]: from='client.? 172.21.15.89:0/342944878' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-21T04:05:27.358 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:05:26 smithi195 ceph-mon[25840]: pgmap v77: 1 pgs: 1 creating+activating; 0 B data, 16 MiB used, 268 GiB / 268 GiB avail 2024-09-21T04:05:27.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:05:26 smithi195 ceph-mon[25840]: from='client.? 172.21.15.89:0/1170865039' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "288ca940-ca66-41f1-894f-32ac0c9d9eb3"}]: dispatch 2024-09-21T04:05:27.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:05:26 smithi195 ceph-mon[25840]: from='client.? 172.21.15.89:0/1170865039' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "288ca940-ca66-41f1-894f-32ac0c9d9eb3"}]': finished 2024-09-21T04:05:27.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:05:26 smithi195 ceph-mon[25840]: osdmap e24: 4 total, 3 up, 4 in 2024-09-21T04:05:27.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:05:26 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-21T04:05:27.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:05:26 smithi195 ceph-mon[25840]: from='client.? 172.21.15.89:0/342944878' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-21T04:05:29.226 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:05:28 smithi089 ceph-mon[23733]: pgmap v79: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail 2024-09-21T04:05:29.358 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:05:28 smithi195 ceph-mon[25840]: pgmap v79: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail 2024-09-21T04:05:31.264 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:05:30 smithi089 ceph-mon[23733]: pgmap v80: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail 2024-09-21T04:05:31.358 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:05:30 smithi195 ceph-mon[25840]: pgmap v80: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail 2024-09-21T04:05:32.934 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:05:32 smithi089 ceph-mon[23733]: pgmap v81: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail 2024-09-21T04:05:32.934 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:05:32 smithi089 ceph-mon[23733]: Detected new or changed devices on smithi089 2024-09-21T04:05:32.934 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:05:32 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' 2024-09-21T04:05:32.934 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:05:32 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-21T04:05:32.934 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:05:32 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-21T04:05:32.934 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:05:32 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-21T04:05:32.934 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:05:32 smithi089 ceph-mon[23733]: Adjusting osd_memory_target on smithi089 to 4287M 2024-09-21T04:05:32.934 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:05:32 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' 2024-09-21T04:05:32.934 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:05:32 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' 2024-09-21T04:05:32.934 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:05:32 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T04:05:32.935 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:05:32 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T04:05:32.935 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:05:32 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T04:05:33.109 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:05:32 smithi195 ceph-mon[25840]: pgmap v81: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail 2024-09-21T04:05:33.109 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:05:32 smithi195 ceph-mon[25840]: Detected new or changed devices on smithi089 2024-09-21T04:05:33.109 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:05:32 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' 2024-09-21T04:05:33.109 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:05:32 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-21T04:05:33.109 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:05:32 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-21T04:05:33.109 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:05:32 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-21T04:05:33.109 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:05:32 smithi195 ceph-mon[25840]: Adjusting osd_memory_target on smithi089 to 4287M 2024-09-21T04:05:33.109 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:05:32 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' 2024-09-21T04:05:33.109 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:05:32 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' 2024-09-21T04:05:33.109 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:05:32 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T04:05:33.109 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:05:32 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T04:05:33.109 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:05:32 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T04:05:33.911 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:05:33 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-09-21T04:05:33.912 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:05:33 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T04:05:34.108 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:05:33 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-09-21T04:05:34.108 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:05:33 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T04:05:35.108 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:05:34 smithi195 ceph-mon[25840]: pgmap v82: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail 2024-09-21T04:05:35.108 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:05:34 smithi195 ceph-mon[25840]: Deploying daemon osd.3 on smithi089 2024-09-21T04:05:35.264 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:05:34 smithi089 ceph-mon[23733]: pgmap v82: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail 2024-09-21T04:05:35.264 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:05:34 smithi089 ceph-mon[23733]: Deploying daemon osd.3 on smithi089 2024-09-21T04:05:37.014 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:05:36 smithi089 ceph-mon[23733]: pgmap v83: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail 2024-09-21T04:05:37.108 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:05:36 smithi195 ceph-mon[25840]: pgmap v83: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail 2024-09-21T04:05:39.014 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:05:38 smithi089 ceph-mon[23733]: pgmap v84: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail 2024-09-21T04:05:39.014 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:05:38 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' 2024-09-21T04:05:39.014 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:05:38 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' 2024-09-21T04:05:39.108 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:05:38 smithi195 ceph-mon[25840]: pgmap v84: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail 2024-09-21T04:05:39.109 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:05:38 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' 2024-09-21T04:05:39.109 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:05:38 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' 2024-09-21T04:05:40.858 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:05:40 smithi195 ceph-mon[25840]: pgmap v85: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail 2024-09-21T04:05:40.859 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:05:40 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' 2024-09-21T04:05:40.859 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:05:40 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T04:05:40.859 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:05:40 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T04:05:40.859 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:05:40 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T04:05:41.005 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:05:40 smithi089 ceph-mon[23733]: pgmap v85: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail 2024-09-21T04:05:41.005 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:05:40 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' 2024-09-21T04:05:41.005 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:05:40 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T04:05:41.005 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:05:40 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T04:05:41.006 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:05:40 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T04:05:42.857 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:05:42 smithi089 ceph-mon[23733]: pgmap v86: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail 2024-09-21T04:05:43.108 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:05:42 smithi195 ceph-mon[25840]: pgmap v86: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail 2024-09-21T04:05:44.015 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:05:43 smithi089 ceph-mon[23733]: from='osd.3 [v2:172.21.15.89:6826/2601107822,v1:172.21.15.89:6827/2601107822]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-09-21T04:05:44.110 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:05:43 smithi195 ceph-mon[25840]: from='osd.3 [v2:172.21.15.89:6826/2601107822,v1:172.21.15.89:6827/2601107822]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-09-21T04:05:44.140 INFO:teuthology.orchestra.run.smithi089.stdout:Created osd(s) 3 on host 'smithi089' 2024-09-21T04:05:45.069 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:05:44 smithi089 ceph-mon[23733]: pgmap v87: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail 2024-09-21T04:05:45.069 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:05:44 smithi089 ceph-mon[23733]: from='osd.3 [v2:172.21.15.89:6826/2601107822,v1:172.21.15.89:6827/2601107822]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-09-21T04:05:45.070 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:05:44 smithi089 ceph-mon[23733]: osdmap e25: 4 total, 3 up, 4 in 2024-09-21T04:05:45.070 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:05:44 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-21T04:05:45.070 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:05:44 smithi089 ceph-mon[23733]: from='osd.3 [v2:172.21.15.89:6826/2601107822,v1:172.21.15.89:6827/2601107822]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi089", "root=default"]}]: dispatch 2024-09-21T04:05:45.070 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:05:44 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' 2024-09-21T04:05:45.092 DEBUG:teuthology.orchestra.run.smithi089:osd.3> sudo journalctl -f -n 0 -u ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490@osd.3.service 2024-09-21T04:05:45.096 INFO:tasks.cephadm:Deploying osd.4 on smithi195 with /dev/vg_nvme/lv_4... 2024-09-21T04:05:45.096 DEBUG:teuthology.orchestra.run.smithi195:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 1b9b79fc-77ce-11ef-baf5-efdc52797490 -- lvm zap /dev/vg_nvme/lv_4 2024-09-21T04:05:45.109 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:05:44 smithi195 ceph-mon[25840]: pgmap v87: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail 2024-09-21T04:05:45.109 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:05:44 smithi195 ceph-mon[25840]: from='osd.3 [v2:172.21.15.89:6826/2601107822,v1:172.21.15.89:6827/2601107822]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-09-21T04:05:45.109 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:05:44 smithi195 ceph-mon[25840]: osdmap e25: 4 total, 3 up, 4 in 2024-09-21T04:05:45.109 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:05:44 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-21T04:05:45.109 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:05:44 smithi195 ceph-mon[25840]: from='osd.3 [v2:172.21.15.89:6826/2601107822,v1:172.21.15.89:6827/2601107822]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi089", "root=default"]}]: dispatch 2024-09-21T04:05:45.110 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:05:44 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' 2024-09-21T04:05:46.109 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:05:45 smithi195 ceph-mon[25840]: from='osd.3 [v2:172.21.15.89:6826/2601107822,v1:172.21.15.89:6827/2601107822]' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi089", "root=default"]}]': finished 2024-09-21T04:05:46.109 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:05:45 smithi195 ceph-mon[25840]: osdmap e26: 4 total, 3 up, 4 in 2024-09-21T04:05:46.109 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:05:45 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-21T04:05:46.109 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:05:45 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-21T04:05:46.170 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:05:45 smithi089 ceph-mon[23733]: from='osd.3 [v2:172.21.15.89:6826/2601107822,v1:172.21.15.89:6827/2601107822]' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi089", "root=default"]}]': finished 2024-09-21T04:05:46.170 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:05:45 smithi089 ceph-mon[23733]: osdmap e26: 4 total, 3 up, 4 in 2024-09-21T04:05:46.170 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:05:45 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-21T04:05:46.170 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:05:45 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-21T04:05:47.014 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:05:46 smithi089 ceph-mon[23733]: purged_snaps scrub starts 2024-09-21T04:05:47.014 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:05:46 smithi089 ceph-mon[23733]: purged_snaps scrub ok 2024-09-21T04:05:47.014 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:05:46 smithi089 ceph-mon[23733]: pgmap v90: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail 2024-09-21T04:05:47.014 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:05:46 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-21T04:05:47.015 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:05:46 smithi089 ceph-mon[23733]: osd.3 [v2:172.21.15.89:6826/2601107822,v1:172.21.15.89:6827/2601107822] boot 2024-09-21T04:05:47.015 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:05:46 smithi089 ceph-mon[23733]: osdmap e27: 4 total, 4 up, 4 in 2024-09-21T04:05:47.015 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:05:46 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-21T04:05:47.102 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:05:46 smithi195 ceph-mon[25840]: purged_snaps scrub starts 2024-09-21T04:05:47.103 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:05:46 smithi195 ceph-mon[25840]: purged_snaps scrub ok 2024-09-21T04:05:47.103 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:05:46 smithi195 ceph-mon[25840]: pgmap v90: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail 2024-09-21T04:05:47.103 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:05:46 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-21T04:05:47.103 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:05:46 smithi195 ceph-mon[25840]: osd.3 [v2:172.21.15.89:6826/2601107822,v1:172.21.15.89:6827/2601107822] boot 2024-09-21T04:05:47.103 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:05:46 smithi195 ceph-mon[25840]: osdmap e27: 4 total, 4 up, 4 in 2024-09-21T04:05:47.103 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:05:46 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-21T04:05:47.726 INFO:teuthology.orchestra.run.smithi195.stdout: 2024-09-21T04:05:47.746 DEBUG:teuthology.orchestra.run.smithi195:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 1b9b79fc-77ce-11ef-baf5-efdc52797490 -- ceph orch daemon add osd smithi195:vg_nvme/lv_4 2024-09-21T04:05:47.945 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:05:47 smithi089 ceph-mon[23733]: osdmap e28: 4 total, 4 up, 4 in 2024-09-21T04:05:47.945 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:05:47 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' 2024-09-21T04:05:48.087 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:05:47 smithi195 ceph-mon[25840]: osdmap e28: 4 total, 4 up, 4 in 2024-09-21T04:05:48.087 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:05:47 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' 2024-09-21T04:05:48.864 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:05:48 smithi089 ceph-mon[23733]: pgmap v93: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-21T04:05:49.082 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:05:48 smithi195 ceph-mon[25840]: pgmap v93: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-21T04:05:50.014 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:05:49 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-21T04:05:50.014 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:05:49 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-21T04:05:50.015 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:05:49 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T04:05:50.090 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:05:49 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-21T04:05:50.090 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:05:49 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-21T04:05:50.090 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:05:49 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T04:05:51.009 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:05:50 smithi195 ceph-mon[25840]: pgmap v94: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-21T04:05:51.009 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:05:50 smithi195 ceph-mon[25840]: from='client.24147 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi195:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T04:05:51.010 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:05:50 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' 2024-09-21T04:05:51.010 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:05:50 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-21T04:05:51.010 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:05:50 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-21T04:05:51.010 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:05:50 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-21T04:05:51.010 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:05:50 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-21T04:05:51.010 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:05:50 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' 2024-09-21T04:05:51.010 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:05:50 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' 2024-09-21T04:05:51.010 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:05:50 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T04:05:51.010 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:05:50 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T04:05:51.010 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:05:50 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T04:05:51.088 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:05:50 smithi089 ceph-mon[23733]: pgmap v94: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-21T04:05:51.088 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:05:50 smithi089 ceph-mon[23733]: from='client.24147 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi195:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T04:05:51.088 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:05:50 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' 2024-09-21T04:05:51.088 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:05:50 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-21T04:05:51.089 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:05:50 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-21T04:05:51.089 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:05:50 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-21T04:05:51.089 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:05:50 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-21T04:05:51.089 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:05:50 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' 2024-09-21T04:05:51.089 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:05:50 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' 2024-09-21T04:05:51.089 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:05:50 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T04:05:51.089 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:05:50 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T04:05:51.089 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:05:50 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T04:05:52.015 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:05:51 smithi089 ceph-mon[23733]: Adjusting osd_memory_target on smithi089 to 3215M 2024-09-21T04:05:52.015 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:05:51 smithi089 ceph-mon[23733]: from='client.? 172.21.15.195:0/2052012500' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "b35557d2-875d-420b-ac49-a0cfe914822d"}]: dispatch 2024-09-21T04:05:52.015 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:05:51 smithi089 ceph-mon[23733]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "b35557d2-875d-420b-ac49-a0cfe914822d"}]: dispatch 2024-09-21T04:05:52.015 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:05:51 smithi089 ceph-mon[23733]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "b35557d2-875d-420b-ac49-a0cfe914822d"}]': finished 2024-09-21T04:05:52.015 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:05:51 smithi089 ceph-mon[23733]: osdmap e29: 5 total, 4 up, 5 in 2024-09-21T04:05:52.015 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:05:51 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-21T04:05:52.109 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:05:51 smithi195 ceph-mon[25840]: Adjusting osd_memory_target on smithi089 to 3215M 2024-09-21T04:05:52.109 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:05:51 smithi195 ceph-mon[25840]: from='client.? 172.21.15.195:0/2052012500' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "b35557d2-875d-420b-ac49-a0cfe914822d"}]: dispatch 2024-09-21T04:05:52.109 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:05:51 smithi195 ceph-mon[25840]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "b35557d2-875d-420b-ac49-a0cfe914822d"}]: dispatch 2024-09-21T04:05:52.109 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:05:51 smithi195 ceph-mon[25840]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "b35557d2-875d-420b-ac49-a0cfe914822d"}]': finished 2024-09-21T04:05:52.109 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:05:51 smithi195 ceph-mon[25840]: osdmap e29: 5 total, 4 up, 5 in 2024-09-21T04:05:52.109 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:05:51 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-21T04:05:53.079 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:05:52 smithi089 ceph-mon[23733]: pgmap v95: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-21T04:05:53.080 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:05:52 smithi089 ceph-mon[23733]: from='client.? 172.21.15.195:0/3583996463' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-21T04:05:53.109 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:05:52 smithi195 ceph-mon[25840]: pgmap v95: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-21T04:05:53.109 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:05:52 smithi195 ceph-mon[25840]: from='client.? 172.21.15.195:0/3583996463' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-21T04:05:55.109 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:05:54 smithi195 ceph-mon[25840]: pgmap v97: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-21T04:05:55.109 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:05:54 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' 2024-09-21T04:05:55.109 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:05:54 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' 2024-09-21T04:05:55.264 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:05:54 smithi089 ceph-mon[23733]: pgmap v97: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-21T04:05:55.264 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:05:54 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' 2024-09-21T04:05:55.264 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:05:54 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' 2024-09-21T04:05:57.074 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:05:56 smithi195 ceph-mon[25840]: pgmap v98: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-21T04:05:57.264 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:05:56 smithi089 ceph-mon[23733]: pgmap v98: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-21T04:05:58.355 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:05:58 smithi195 ceph-mon[25840]: pgmap v99: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-21T04:05:58.356 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:05:58 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-09-21T04:05:58.356 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:05:58 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T04:05:58.356 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:05:58 smithi195 ceph-mon[25840]: Deploying daemon osd.4 on smithi195 2024-09-21T04:05:58.514 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:05:58 smithi089 ceph-mon[23733]: pgmap v99: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-21T04:05:58.514 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:05:58 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-09-21T04:05:58.515 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:05:58 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T04:05:58.515 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:05:58 smithi089 ceph-mon[23733]: Deploying daemon osd.4 on smithi195 2024-09-21T04:06:00.358 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:06:00 smithi195 ceph-mon[25840]: pgmap v100: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-21T04:06:00.514 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:06:00 smithi089 ceph-mon[23733]: pgmap v100: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-21T04:06:02.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:06:02 smithi195 ceph-mon[25840]: pgmap v101: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-21T04:06:02.514 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:06:02 smithi089 ceph-mon[23733]: pgmap v101: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-21T04:06:03.859 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:06:03 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' 2024-09-21T04:06:03.859 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:06:03 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T04:06:03.859 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:06:03 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T04:06:03.859 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:06:03 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T04:06:04.015 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:06:03 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' 2024-09-21T04:06:04.015 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:06:03 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T04:06:04.015 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:06:03 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T04:06:04.015 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:06:03 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T04:06:05.014 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:06:04 smithi089 ceph-mon[23733]: pgmap v102: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-21T04:06:05.109 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:06:04 smithi195 ceph-mon[25840]: pgmap v102: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-21T04:06:05.898 INFO:teuthology.orchestra.run.smithi195.stdout:Created osd(s) 4 on host 'smithi195' 2024-09-21T04:06:06.474 DEBUG:teuthology.orchestra.run.smithi195:osd.4> sudo journalctl -f -n 0 -u ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490@osd.4.service 2024-09-21T04:06:06.476 INFO:tasks.cephadm:Deploying osd.5 on smithi195 with /dev/vg_nvme/lv_3... 2024-09-21T04:06:06.476 DEBUG:teuthology.orchestra.run.smithi195:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 1b9b79fc-77ce-11ef-baf5-efdc52797490 -- lvm zap /dev/vg_nvme/lv_3 2024-09-21T04:06:06.832 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:06:06 smithi195 ceph-mon[25840]: pgmap v103: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-21T04:06:06.832 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:06:06 smithi195 ceph-mon[25840]: from='osd.4 [v2:172.21.15.195:6800/44100003,v1:172.21.15.195:6801/44100003]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-09-21T04:06:06.832 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:06:06 smithi195 ceph-mon[25840]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-09-21T04:06:06.832 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:06:06 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' 2024-09-21T04:06:07.014 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:06:06 smithi089 ceph-mon[23733]: pgmap v103: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-21T04:06:07.014 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:06:06 smithi089 ceph-mon[23733]: from='osd.4 [v2:172.21.15.195:6800/44100003,v1:172.21.15.195:6801/44100003]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-09-21T04:06:07.015 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:06:06 smithi089 ceph-mon[23733]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-09-21T04:06:07.015 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:06:06 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' 2024-09-21T04:06:07.853 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:06:07 smithi195 ceph-mon[25840]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-09-21T04:06:07.853 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:06:07 smithi195 ceph-mon[25840]: osdmap e30: 5 total, 4 up, 5 in 2024-09-21T04:06:07.853 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:06:07 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-21T04:06:07.853 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:06:07 smithi195 ceph-mon[25840]: from='osd.4 [v2:172.21.15.195:6800/44100003,v1:172.21.15.195:6801/44100003]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi195", "root=default"]}]: dispatch 2024-09-21T04:06:07.853 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:06:07 smithi195 ceph-mon[25840]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi195", "root=default"]}]: dispatch 2024-09-21T04:06:07.854 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:06:07 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' 2024-09-21T04:06:07.854 INFO:journalctl@ceph.osd.4.smithi195.stdout:Sep 21 04:06:07 smithi195 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-4[30043]: 2024-09-21T04:06:07.660+0000 7fa19e430700 -1 osd.4 0 waiting for initial osdmap 2024-09-21T04:06:07.854 INFO:journalctl@ceph.osd.4.smithi195.stdout:Sep 21 04:06:07 smithi195 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-4[30043]: 2024-09-21T04:06:07.664+0000 7fa19adcb700 -1 osd.4 31 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-21T04:06:08.014 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:06:07 smithi089 ceph-mon[23733]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-09-21T04:06:08.014 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:06:07 smithi089 ceph-mon[23733]: osdmap e30: 5 total, 4 up, 5 in 2024-09-21T04:06:08.015 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:06:07 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-21T04:06:08.015 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:06:07 smithi089 ceph-mon[23733]: from='osd.4 [v2:172.21.15.195:6800/44100003,v1:172.21.15.195:6801/44100003]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi195", "root=default"]}]: dispatch 2024-09-21T04:06:08.015 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:06:07 smithi089 ceph-mon[23733]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi195", "root=default"]}]: dispatch 2024-09-21T04:06:08.015 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:06:07 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' 2024-09-21T04:06:08.842 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:06:08 smithi195 ceph-mon[25840]: pgmap v105: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-21T04:06:08.843 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:06:08 smithi195 ceph-mon[25840]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi195", "root=default"]}]': finished 2024-09-21T04:06:08.843 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:06:08 smithi195 ceph-mon[25840]: osdmap e31: 5 total, 4 up, 5 in 2024-09-21T04:06:08.843 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:06:08 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-21T04:06:08.843 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:06:08 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-21T04:06:09.014 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:06:08 smithi089 ceph-mon[23733]: pgmap v105: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-21T04:06:09.015 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:06:08 smithi089 ceph-mon[23733]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi195", "root=default"]}]': finished 2024-09-21T04:06:09.015 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:06:08 smithi089 ceph-mon[23733]: osdmap e31: 5 total, 4 up, 5 in 2024-09-21T04:06:09.015 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:06:08 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-21T04:06:09.015 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:06:08 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-21T04:06:09.859 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:06:09 smithi195 ceph-mon[25840]: purged_snaps scrub starts 2024-09-21T04:06:09.859 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:06:09 smithi195 ceph-mon[25840]: purged_snaps scrub ok 2024-09-21T04:06:09.859 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:06:09 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-21T04:06:09.859 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:06:09 smithi195 ceph-mon[25840]: osd.4 [v2:172.21.15.195:6800/44100003,v1:172.21.15.195:6801/44100003] boot 2024-09-21T04:06:09.859 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:06:09 smithi195 ceph-mon[25840]: osdmap e32: 5 total, 5 up, 5 in 2024-09-21T04:06:09.859 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:06:09 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-21T04:06:09.859 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:06:09 smithi195 ceph-mon[25840]: osdmap e33: 5 total, 5 up, 5 in 2024-09-21T04:06:09.896 INFO:teuthology.orchestra.run.smithi195.stdout: 2024-09-21T04:06:09.920 DEBUG:teuthology.orchestra.run.smithi195:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 1b9b79fc-77ce-11ef-baf5-efdc52797490 -- ceph orch daemon add osd smithi195:vg_nvme/lv_3 2024-09-21T04:06:10.014 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:06:09 smithi089 ceph-mon[23733]: purged_snaps scrub starts 2024-09-21T04:06:10.015 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:06:09 smithi089 ceph-mon[23733]: purged_snaps scrub ok 2024-09-21T04:06:10.015 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:06:09 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-21T04:06:10.015 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:06:09 smithi089 ceph-mon[23733]: osd.4 [v2:172.21.15.195:6800/44100003,v1:172.21.15.195:6801/44100003] boot 2024-09-21T04:06:10.015 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:06:09 smithi089 ceph-mon[23733]: osdmap e32: 5 total, 5 up, 5 in 2024-09-21T04:06:10.015 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:06:09 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-21T04:06:10.015 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:06:09 smithi089 ceph-mon[23733]: osdmap e33: 5 total, 5 up, 5 in 2024-09-21T04:06:10.790 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:06:10 smithi195 ceph-mon[25840]: pgmap v108: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-21T04:06:10.790 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:06:10 smithi195 ceph-mon[25840]: osdmap e34: 5 total, 5 up, 5 in 2024-09-21T04:06:11.014 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:06:10 smithi089 ceph-mon[23733]: pgmap v108: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-21T04:06:11.014 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:06:10 smithi089 ceph-mon[23733]: osdmap e34: 5 total, 5 up, 5 in 2024-09-21T04:06:12.844 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:06:12 smithi195 ceph-mon[25840]: pgmap v111: 1 pgs: 1 active+clean; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail 2024-09-21T04:06:12.844 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:06:12 smithi195 ceph-mon[25840]: osdmap e35: 5 total, 5 up, 5 in 2024-09-21T04:06:12.844 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:06:12 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' 2024-09-21T04:06:12.844 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:06:12 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-21T04:06:12.844 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:06:12 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' 2024-09-21T04:06:12.844 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:06:12 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' 2024-09-21T04:06:12.845 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:06:12 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T04:06:12.845 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:06:12 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T04:06:12.845 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:06:12 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T04:06:12.845 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:06:12 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' 2024-09-21T04:06:13.014 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:06:12 smithi089 ceph-mon[23733]: pgmap v111: 1 pgs: 1 active+clean; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail 2024-09-21T04:06:13.015 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:06:12 smithi089 ceph-mon[23733]: osdmap e35: 5 total, 5 up, 5 in 2024-09-21T04:06:13.015 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:06:12 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' 2024-09-21T04:06:13.015 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:06:12 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-21T04:06:13.015 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:06:12 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' 2024-09-21T04:06:13.015 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:06:12 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' 2024-09-21T04:06:13.015 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:06:12 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T04:06:13.015 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:06:12 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T04:06:13.015 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:06:12 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T04:06:13.015 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:06:12 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' 2024-09-21T04:06:13.848 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:06:13 smithi195 ceph-mon[25840]: Detected new or changed devices on smithi195 2024-09-21T04:06:13.848 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:06:13 smithi195 ceph-mon[25840]: Adjusting osd_memory_target on smithi195 to 15936M 2024-09-21T04:06:13.848 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:06:13 smithi195 ceph-mon[25840]: osdmap e36: 5 total, 5 up, 5 in 2024-09-21T04:06:13.848 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:06:13 smithi195 ceph-mon[25840]: from='client.24165 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi195:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T04:06:13.848 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:06:13 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-21T04:06:13.848 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:06:13 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-21T04:06:13.848 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:06:13 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T04:06:13.848 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:06:13 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi089.srlyxf/trash_purge_schedule"}]: dispatch 2024-09-21T04:06:13.848 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:06:13 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi089.srlyxf/mirror_snapshot_schedule"}]: dispatch 2024-09-21T04:06:14.015 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:06:13 smithi089 ceph-mon[23733]: Detected new or changed devices on smithi195 2024-09-21T04:06:14.015 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:06:13 smithi089 ceph-mon[23733]: Adjusting osd_memory_target on smithi195 to 15936M 2024-09-21T04:06:14.015 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:06:13 smithi089 ceph-mon[23733]: osdmap e36: 5 total, 5 up, 5 in 2024-09-21T04:06:14.015 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:06:13 smithi089 ceph-mon[23733]: from='client.24165 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi195:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T04:06:14.015 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:06:13 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-21T04:06:14.015 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:06:13 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-21T04:06:14.015 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:06:13 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T04:06:14.015 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:06:13 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi089.srlyxf/trash_purge_schedule"}]: dispatch 2024-09-21T04:06:14.015 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:06:13 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi089.srlyxf/mirror_snapshot_schedule"}]: dispatch 2024-09-21T04:06:14.859 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:06:14 smithi195 ceph-mon[25840]: pgmap v114: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 129 KiB/s, 0 objects/s recovering 2024-09-21T04:06:15.014 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:06:14 smithi089 ceph-mon[23733]: pgmap v114: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 129 KiB/s, 0 objects/s recovering 2024-09-21T04:06:16.015 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:06:15 smithi089 ceph-mon[23733]: from='client.? 172.21.15.195:0/2108876837' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "94ffbc65-1d43-460b-84eb-6be843a72bae"}]: dispatch 2024-09-21T04:06:16.015 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:06:15 smithi089 ceph-mon[23733]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "94ffbc65-1d43-460b-84eb-6be843a72bae"}]: dispatch 2024-09-21T04:06:16.015 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:06:15 smithi089 ceph-mon[23733]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "94ffbc65-1d43-460b-84eb-6be843a72bae"}]': finished 2024-09-21T04:06:16.015 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:06:15 smithi089 ceph-mon[23733]: osdmap e37: 6 total, 5 up, 6 in 2024-09-21T04:06:16.015 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:06:15 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-21T04:06:16.108 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:06:15 smithi195 ceph-mon[25840]: from='client.? 172.21.15.195:0/2108876837' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "94ffbc65-1d43-460b-84eb-6be843a72bae"}]: dispatch 2024-09-21T04:06:16.109 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:06:15 smithi195 ceph-mon[25840]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "94ffbc65-1d43-460b-84eb-6be843a72bae"}]: dispatch 2024-09-21T04:06:16.109 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:06:15 smithi195 ceph-mon[25840]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "94ffbc65-1d43-460b-84eb-6be843a72bae"}]': finished 2024-09-21T04:06:16.109 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:06:15 smithi195 ceph-mon[25840]: osdmap e37: 6 total, 5 up, 6 in 2024-09-21T04:06:16.109 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:06:15 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-21T04:06:17.015 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:06:16 smithi089 ceph-mon[23733]: pgmap v115: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 95 KiB/s, 0 objects/s recovering 2024-09-21T04:06:17.015 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:06:16 smithi089 ceph-mon[23733]: from='client.? 172.21.15.195:0/3509957879' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-21T04:06:17.109 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:06:16 smithi195 ceph-mon[25840]: pgmap v115: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 95 KiB/s, 0 objects/s recovering 2024-09-21T04:06:17.109 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:06:16 smithi195 ceph-mon[25840]: from='client.? 172.21.15.195:0/3509957879' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-21T04:06:18.859 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:06:18 smithi195 ceph-mon[25840]: pgmap v117: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-09-21T04:06:19.014 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:06:18 smithi089 ceph-mon[23733]: pgmap v117: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-09-21T04:06:20.828 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:06:20 smithi195 ceph-mon[25840]: pgmap v118: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-21T04:06:21.015 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:06:20 smithi089 ceph-mon[23733]: pgmap v118: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-21T04:06:21.731 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:06:21 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-09-21T04:06:21.731 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:06:21 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T04:06:22.014 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:06:21 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-09-21T04:06:22.014 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:06:21 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T04:06:22.859 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:06:22 smithi195 ceph-mon[25840]: Deploying daemon osd.5 on smithi195 2024-09-21T04:06:22.859 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:06:22 smithi195 ceph-mon[25840]: pgmap v119: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-21T04:06:23.014 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:06:22 smithi089 ceph-mon[23733]: Deploying daemon osd.5 on smithi195 2024-09-21T04:06:23.015 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:06:22 smithi089 ceph-mon[23733]: pgmap v119: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-21T04:06:25.015 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:06:24 smithi089 ceph-mon[23733]: pgmap v120: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-21T04:06:25.072 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:06:24 smithi195 ceph-mon[25840]: pgmap v120: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-21T04:06:27.264 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:06:27 smithi089 ceph-mon[23733]: pgmap v121: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-21T04:06:27.264 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:06:27 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' 2024-09-21T04:06:27.265 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:06:27 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T04:06:27.265 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:06:27 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T04:06:27.265 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:06:27 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T04:06:27.358 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:06:27 smithi195 ceph-mon[25840]: pgmap v121: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-21T04:06:27.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:06:27 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' 2024-09-21T04:06:27.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:06:27 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T04:06:27.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:06:27 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T04:06:27.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:06:27 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T04:06:29.264 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:06:29 smithi089 ceph-mon[23733]: pgmap v122: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-21T04:06:29.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:06:29 smithi195 ceph-mon[25840]: pgmap v122: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-21T04:06:29.487 INFO:teuthology.orchestra.run.smithi195.stdout:Created osd(s) 5 on host 'smithi195' 2024-09-21T04:06:30.264 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:06:30 smithi089 ceph-mon[23733]: from='osd.5 [v2:172.21.15.195:6808/1913588562,v1:172.21.15.195:6809/1913588562]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-09-21T04:06:30.265 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:06:30 smithi089 ceph-mon[23733]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-09-21T04:06:30.265 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:06:30 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' 2024-09-21T04:06:30.265 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:06:30 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' 2024-09-21T04:06:30.291 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:06:30 smithi195 ceph-mon[25840]: from='osd.5 [v2:172.21.15.195:6808/1913588562,v1:172.21.15.195:6809/1913588562]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-09-21T04:06:30.292 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:06:30 smithi195 ceph-mon[25840]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-09-21T04:06:30.292 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:06:30 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' 2024-09-21T04:06:30.292 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:06:30 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' 2024-09-21T04:06:30.314 DEBUG:teuthology.orchestra.run.smithi195:osd.5> sudo journalctl -f -n 0 -u ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490@osd.5.service 2024-09-21T04:06:30.316 INFO:tasks.cephadm:Deploying osd.6 on smithi195 with /dev/vg_nvme/lv_2... 2024-09-21T04:06:30.316 DEBUG:teuthology.orchestra.run.smithi195:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 1b9b79fc-77ce-11ef-baf5-efdc52797490 -- lvm zap /dev/vg_nvme/lv_2 2024-09-21T04:06:31.264 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:06:31 smithi089 ceph-mon[23733]: pgmap v123: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-21T04:06:31.264 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:06:31 smithi089 ceph-mon[23733]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-09-21T04:06:31.264 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:06:31 smithi089 ceph-mon[23733]: osdmap e38: 6 total, 5 up, 6 in 2024-09-21T04:06:31.264 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:06:31 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-21T04:06:31.265 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:06:31 smithi089 ceph-mon[23733]: from='osd.5 [v2:172.21.15.195:6808/1913588562,v1:172.21.15.195:6809/1913588562]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi195", "root=default"]}]: dispatch 2024-09-21T04:06:31.265 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:06:31 smithi089 ceph-mon[23733]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi195", "root=default"]}]: dispatch 2024-09-21T04:06:31.299 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:06:31 smithi195 ceph-mon[25840]: pgmap v123: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-21T04:06:31.299 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:06:31 smithi195 ceph-mon[25840]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-09-21T04:06:31.299 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:06:31 smithi195 ceph-mon[25840]: osdmap e38: 6 total, 5 up, 6 in 2024-09-21T04:06:31.299 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:06:31 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-21T04:06:31.299 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:06:31 smithi195 ceph-mon[25840]: from='osd.5 [v2:172.21.15.195:6808/1913588562,v1:172.21.15.195:6809/1913588562]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi195", "root=default"]}]: dispatch 2024-09-21T04:06:31.299 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:06:31 smithi195 ceph-mon[25840]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi195", "root=default"]}]: dispatch 2024-09-21T04:06:31.299 INFO:journalctl@ceph.osd.5.smithi195.stdout:Sep 21 04:06:31 smithi195 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-5[32992]: 2024-09-21T04:06:31.014+0000 7fdedfdb4700 -1 osd.5 0 waiting for initial osdmap 2024-09-21T04:06:31.299 INFO:journalctl@ceph.osd.5.smithi195.stdout:Sep 21 04:06:31 smithi195 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-5[32992]: 2024-09-21T04:06:31.019+0000 7fded8f48700 -1 osd.5 39 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-21T04:06:32.188 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:06:32 smithi195 ceph-mon[25840]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi195", "root=default"]}]': finished 2024-09-21T04:06:32.188 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:06:32 smithi195 ceph-mon[25840]: osdmap e39: 6 total, 5 up, 6 in 2024-09-21T04:06:32.188 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:06:32 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-21T04:06:32.188 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:06:32 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-21T04:06:32.514 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:06:32 smithi089 ceph-mon[23733]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi195", "root=default"]}]': finished 2024-09-21T04:06:32.514 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:06:32 smithi089 ceph-mon[23733]: osdmap e39: 6 total, 5 up, 6 in 2024-09-21T04:06:32.515 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:06:32 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-21T04:06:32.515 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:06:32 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-21T04:06:33.264 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:06:33 smithi089 ceph-mon[23733]: purged_snaps scrub starts 2024-09-21T04:06:33.264 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:06:33 smithi089 ceph-mon[23733]: purged_snaps scrub ok 2024-09-21T04:06:33.264 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:06:33 smithi089 ceph-mon[23733]: pgmap v126: 1 pgs: 1 peering; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-21T04:06:33.265 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:06:33 smithi089 ceph-mon[23733]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-09-21T04:06:33.265 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:06:33 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-21T04:06:33.265 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:06:33 smithi089 ceph-mon[23733]: osd.5 [v2:172.21.15.195:6808/1913588562,v1:172.21.15.195:6809/1913588562] boot 2024-09-21T04:06:33.265 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:06:33 smithi089 ceph-mon[23733]: osdmap e40: 6 total, 6 up, 6 in 2024-09-21T04:06:33.265 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:06:33 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-21T04:06:33.265 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:06:33 smithi089 ceph-mon[23733]: osdmap e41: 6 total, 6 up, 6 in 2024-09-21T04:06:33.338 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:06:33 smithi195 ceph-mon[25840]: purged_snaps scrub starts 2024-09-21T04:06:33.339 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:06:33 smithi195 ceph-mon[25840]: purged_snaps scrub ok 2024-09-21T04:06:33.339 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:06:33 smithi195 ceph-mon[25840]: pgmap v126: 1 pgs: 1 peering; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-21T04:06:33.339 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:06:33 smithi195 ceph-mon[25840]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-09-21T04:06:33.339 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:06:33 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-21T04:06:33.339 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:06:33 smithi195 ceph-mon[25840]: osd.5 [v2:172.21.15.195:6808/1913588562,v1:172.21.15.195:6809/1913588562] boot 2024-09-21T04:06:33.339 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:06:33 smithi195 ceph-mon[25840]: osdmap e40: 6 total, 6 up, 6 in 2024-09-21T04:06:33.339 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:06:33 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-21T04:06:33.339 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:06:33 smithi195 ceph-mon[25840]: osdmap e41: 6 total, 6 up, 6 in 2024-09-21T04:06:34.347 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:06:34 smithi195 ceph-mon[25840]: Detected new or changed devices on smithi195 2024-09-21T04:06:34.347 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:06:34 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' 2024-09-21T04:06:34.347 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:06:34 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-21T04:06:34.347 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:06:34 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-21T04:06:34.347 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:06:34 smithi195 ceph-mon[25840]: Adjusting osd_memory_target on smithi195 to 7968M 2024-09-21T04:06:34.347 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:06:34 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' 2024-09-21T04:06:34.347 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:06:34 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' 2024-09-21T04:06:34.347 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:06:34 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T04:06:34.347 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:06:34 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T04:06:34.347 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:06:34 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T04:06:34.347 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:06:34 smithi195 ceph-mon[25840]: pgmap v129: 1 pgs: 1 peering; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-09-21T04:06:34.348 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:06:34 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' 2024-09-21T04:06:34.348 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:06:34 smithi195 ceph-mon[25840]: osdmap e42: 6 total, 6 up, 6 in 2024-09-21T04:06:34.515 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:06:34 smithi089 ceph-mon[23733]: Detected new or changed devices on smithi195 2024-09-21T04:06:34.515 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:06:34 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' 2024-09-21T04:06:34.515 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:06:34 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-21T04:06:34.515 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:06:34 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-21T04:06:34.515 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:06:34 smithi089 ceph-mon[23733]: Adjusting osd_memory_target on smithi195 to 7968M 2024-09-21T04:06:34.515 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:06:34 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' 2024-09-21T04:06:34.515 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:06:34 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' 2024-09-21T04:06:34.515 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:06:34 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T04:06:34.515 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:06:34 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T04:06:34.515 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:06:34 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T04:06:34.515 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:06:34 smithi089 ceph-mon[23733]: pgmap v129: 1 pgs: 1 peering; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-09-21T04:06:34.516 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:06:34 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' 2024-09-21T04:06:34.516 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:06:34 smithi089 ceph-mon[23733]: osdmap e42: 6 total, 6 up, 6 in 2024-09-21T04:06:34.952 INFO:teuthology.orchestra.run.smithi195.stdout: 2024-09-21T04:06:34.972 DEBUG:teuthology.orchestra.run.smithi195:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 1b9b79fc-77ce-11ef-baf5-efdc52797490 -- ceph orch daemon add osd smithi195:vg_nvme/lv_2 2024-09-21T04:06:35.263 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:06:35 smithi195 ceph-mon[25840]: osdmap e43: 6 total, 6 up, 6 in 2024-09-21T04:06:35.514 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:06:35 smithi089 ceph-mon[23733]: osdmap e43: 6 total, 6 up, 6 in 2024-09-21T04:06:36.426 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:06:36 smithi195 ceph-mon[25840]: pgmap v132: 1 pgs: 1 peering; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-09-21T04:06:36.426 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:06:36 smithi195 ceph-mon[25840]: osdmap e44: 6 total, 6 up, 6 in 2024-09-21T04:06:36.514 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:06:36 smithi089 ceph-mon[23733]: pgmap v132: 1 pgs: 1 peering; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-09-21T04:06:36.514 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:06:36 smithi089 ceph-mon[23733]: osdmap e44: 6 total, 6 up, 6 in 2024-09-21T04:06:37.383 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:06:37 smithi195 ceph-mon[25840]: from='client.24183 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi195:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T04:06:37.383 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:06:37 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-21T04:06:37.383 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:06:37 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-21T04:06:37.383 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:06:37 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T04:06:37.514 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:06:37 smithi089 ceph-mon[23733]: from='client.24183 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi195:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T04:06:37.515 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:06:37 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-21T04:06:37.515 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:06:37 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-21T04:06:37.515 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:06:37 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T04:06:38.429 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:06:38 smithi195 ceph-mon[25840]: pgmap v134: 1 pgs: 1 peering; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-21T04:06:38.514 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:06:38 smithi089 ceph-mon[23733]: pgmap v134: 1 pgs: 1 peering; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-21T04:06:39.514 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:06:39 smithi089 ceph-mon[23733]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-09-21T04:06:39.515 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:06:39 smithi089 ceph-mon[23733]: Cluster is now healthy 2024-09-21T04:06:39.608 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:06:39 smithi195 ceph-mon[25840]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-09-21T04:06:39.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:06:39 smithi195 ceph-mon[25840]: Cluster is now healthy 2024-09-21T04:06:40.514 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:06:40 smithi089 ceph-mon[23733]: pgmap v135: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-09-21T04:06:40.514 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:06:40 smithi089 ceph-mon[23733]: from='client.? 172.21.15.195:0/1778658384' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "3ae73a35-8e39-47b1-8f1b-e52c81bbf9c8"}]: dispatch 2024-09-21T04:06:40.515 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:06:40 smithi089 ceph-mon[23733]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "3ae73a35-8e39-47b1-8f1b-e52c81bbf9c8"}]: dispatch 2024-09-21T04:06:40.515 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:06:40 smithi089 ceph-mon[23733]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "3ae73a35-8e39-47b1-8f1b-e52c81bbf9c8"}]': finished 2024-09-21T04:06:40.515 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:06:40 smithi089 ceph-mon[23733]: osdmap e45: 7 total, 6 up, 7 in 2024-09-21T04:06:40.515 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:06:40 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-21T04:06:40.515 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:06:40 smithi089 ceph-mon[23733]: from='client.? 172.21.15.195:0/2541418424' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-21T04:06:40.596 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:06:40 smithi195 ceph-mon[25840]: pgmap v135: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-09-21T04:06:40.596 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:06:40 smithi195 ceph-mon[25840]: from='client.? 172.21.15.195:0/1778658384' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "3ae73a35-8e39-47b1-8f1b-e52c81bbf9c8"}]: dispatch 2024-09-21T04:06:40.596 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:06:40 smithi195 ceph-mon[25840]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "3ae73a35-8e39-47b1-8f1b-e52c81bbf9c8"}]: dispatch 2024-09-21T04:06:40.596 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:06:40 smithi195 ceph-mon[25840]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "3ae73a35-8e39-47b1-8f1b-e52c81bbf9c8"}]': finished 2024-09-21T04:06:40.596 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:06:40 smithi195 ceph-mon[25840]: osdmap e45: 7 total, 6 up, 7 in 2024-09-21T04:06:40.596 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:06:40 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-21T04:06:40.596 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:06:40 smithi195 ceph-mon[25840]: from='client.? 172.21.15.195:0/2541418424' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-21T04:06:42.328 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:06:42 smithi195 ceph-mon[25840]: pgmap v137: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 74 KiB/s, 0 objects/s recovering 2024-09-21T04:06:42.514 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:06:42 smithi089 ceph-mon[23733]: pgmap v137: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 74 KiB/s, 0 objects/s recovering 2024-09-21T04:06:44.279 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:06:44 smithi195 ceph-mon[25840]: pgmap v138: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-09-21T04:06:44.514 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:06:44 smithi089 ceph-mon[23733]: pgmap v138: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-09-21T04:06:46.515 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:06:46 smithi089 ceph-mon[23733]: pgmap v139: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-09-21T04:06:46.515 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:06:46 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-09-21T04:06:46.515 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:06:46 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T04:06:46.515 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:06:46 smithi089 ceph-mon[23733]: Deploying daemon osd.6 on smithi195 2024-09-21T04:06:46.608 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:06:46 smithi195 ceph-mon[25840]: pgmap v139: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-09-21T04:06:46.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:06:46 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-09-21T04:06:46.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:06:46 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T04:06:46.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:06:46 smithi195 ceph-mon[25840]: Deploying daemon osd.6 on smithi195 2024-09-21T04:06:48.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:06:48 smithi195 ceph-mon[25840]: pgmap v140: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-09-21T04:06:48.514 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:06:48 smithi089 ceph-mon[23733]: pgmap v140: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-09-21T04:06:50.234 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:06:50 smithi195 ceph-mon[25840]: pgmap v141: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-21T04:06:50.514 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:06:50 smithi089 ceph-mon[23733]: pgmap v141: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-21T04:06:51.514 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:06:51 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' 2024-09-21T04:06:51.515 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:06:51 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T04:06:51.515 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:06:51 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T04:06:51.515 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:06:51 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T04:06:51.520 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:06:51 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' 2024-09-21T04:06:51.520 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:06:51 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T04:06:51.520 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:06:51 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T04:06:51.520 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:06:51 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T04:06:52.515 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:06:52 smithi089 ceph-mon[23733]: pgmap v142: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-21T04:06:52.603 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:06:52 smithi195 ceph-mon[25840]: pgmap v142: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-21T04:06:52.920 INFO:teuthology.orchestra.run.smithi195.stdout:Created osd(s) 6 on host 'smithi195' 2024-09-21T04:06:53.927 DEBUG:teuthology.orchestra.run.smithi195:osd.6> sudo journalctl -f -n 0 -u ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490@osd.6.service 2024-09-21T04:06:53.929 INFO:tasks.cephadm:Deploying osd.7 on smithi195 with /dev/vg_nvme/lv_1... 2024-09-21T04:06:53.929 DEBUG:teuthology.orchestra.run.smithi195:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 1b9b79fc-77ce-11ef-baf5-efdc52797490 -- lvm zap /dev/vg_nvme/lv_1 2024-09-21T04:06:54.188 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:06:53 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' 2024-09-21T04:06:54.188 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:06:53 smithi195 ceph-mon[25840]: from='osd.6 [v2:172.21.15.195:6816/1620163911,v1:172.21.15.195:6817/1620163911]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-09-21T04:06:54.189 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:06:53 smithi195 ceph-mon[25840]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-09-21T04:06:54.264 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:06:53 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' 2024-09-21T04:06:54.264 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:06:53 smithi089 ceph-mon[23733]: from='osd.6 [v2:172.21.15.195:6816/1620163911,v1:172.21.15.195:6817/1620163911]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-09-21T04:06:54.265 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:06:53 smithi089 ceph-mon[23733]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-09-21T04:06:55.083 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:06:54 smithi195 ceph-mon[25840]: pgmap v143: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-21T04:06:55.083 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:06:54 smithi195 ceph-mon[25840]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-09-21T04:06:55.084 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:06:54 smithi195 ceph-mon[25840]: osdmap e46: 7 total, 6 up, 7 in 2024-09-21T04:06:55.084 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:06:54 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-21T04:06:55.084 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:06:54 smithi195 ceph-mon[25840]: from='osd.6 [v2:172.21.15.195:6816/1620163911,v1:172.21.15.195:6817/1620163911]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi195", "root=default"]}]: dispatch 2024-09-21T04:06:55.084 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:06:54 smithi195 ceph-mon[25840]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi195", "root=default"]}]: dispatch 2024-09-21T04:06:55.084 INFO:journalctl@ceph.osd.6.smithi195.stdout:Sep 21 04:06:54 smithi195 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-6[35964]: 2024-09-21T04:06:54.928+0000 7fde5aa0b700 -1 osd.6 0 waiting for initial osdmap 2024-09-21T04:06:55.084 INFO:journalctl@ceph.osd.6.smithi195.stdout:Sep 21 04:06:54 smithi195 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-6[35964]: 2024-09-21T04:06:54.934+0000 7fde553a2700 -1 osd.6 47 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-21T04:06:55.264 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:06:54 smithi089 ceph-mon[23733]: pgmap v143: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-21T04:06:55.264 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:06:54 smithi089 ceph-mon[23733]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-09-21T04:06:55.264 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:06:54 smithi089 ceph-mon[23733]: osdmap e46: 7 total, 6 up, 7 in 2024-09-21T04:06:55.265 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:06:54 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-21T04:06:55.265 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:06:54 smithi089 ceph-mon[23733]: from='osd.6 [v2:172.21.15.195:6816/1620163911,v1:172.21.15.195:6817/1620163911]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi195", "root=default"]}]: dispatch 2024-09-21T04:06:55.265 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:06:54 smithi089 ceph-mon[23733]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi195", "root=default"]}]: dispatch 2024-09-21T04:06:56.065 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:06:55 smithi195 ceph-mon[25840]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi195", "root=default"]}]': finished 2024-09-21T04:06:56.065 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:06:55 smithi195 ceph-mon[25840]: osdmap e47: 7 total, 6 up, 7 in 2024-09-21T04:06:56.066 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:06:55 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-21T04:06:56.066 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:06:55 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-21T04:06:56.066 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:06:55 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' 2024-09-21T04:06:56.264 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:06:55 smithi089 ceph-mon[23733]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi195", "root=default"]}]': finished 2024-09-21T04:06:56.265 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:06:55 smithi089 ceph-mon[23733]: osdmap e47: 7 total, 6 up, 7 in 2024-09-21T04:06:56.265 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:06:55 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-21T04:06:56.265 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:06:55 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-21T04:06:56.265 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:06:55 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' 2024-09-21T04:06:57.108 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:06:56 smithi195 ceph-mon[25840]: purged_snaps scrub starts 2024-09-21T04:06:57.109 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:06:56 smithi195 ceph-mon[25840]: purged_snaps scrub ok 2024-09-21T04:06:57.109 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:06:56 smithi195 ceph-mon[25840]: pgmap v146: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-21T04:06:57.109 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:06:56 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-21T04:06:57.109 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:06:56 smithi195 ceph-mon[25840]: osd.6 [v2:172.21.15.195:6816/1620163911,v1:172.21.15.195:6817/1620163911] boot 2024-09-21T04:06:57.109 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:06:56 smithi195 ceph-mon[25840]: osdmap e48: 7 total, 7 up, 7 in 2024-09-21T04:06:57.109 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:06:56 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-21T04:06:57.264 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:06:56 smithi089 ceph-mon[23733]: purged_snaps scrub starts 2024-09-21T04:06:57.264 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:06:56 smithi089 ceph-mon[23733]: purged_snaps scrub ok 2024-09-21T04:06:57.265 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:06:56 smithi089 ceph-mon[23733]: pgmap v146: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-21T04:06:57.265 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:06:56 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-21T04:06:57.265 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:06:56 smithi089 ceph-mon[23733]: osd.6 [v2:172.21.15.195:6816/1620163911,v1:172.21.15.195:6817/1620163911] boot 2024-09-21T04:06:57.265 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:06:56 smithi089 ceph-mon[23733]: osdmap e48: 7 total, 7 up, 7 in 2024-09-21T04:06:57.265 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:06:56 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-21T04:06:57.503 INFO:teuthology.orchestra.run.smithi195.stdout: 2024-09-21T04:06:57.526 DEBUG:teuthology.orchestra.run.smithi195:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 1b9b79fc-77ce-11ef-baf5-efdc52797490 -- ceph orch daemon add osd smithi195:vg_nvme/lv_1 2024-09-21T04:06:58.191 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:06:57 smithi195 ceph-mon[25840]: osdmap e49: 7 total, 7 up, 7 in 2024-09-21T04:06:58.264 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:06:57 smithi089 ceph-mon[23733]: osdmap e49: 7 total, 7 up, 7 in 2024-09-21T04:06:59.264 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:06:58 smithi089 ceph-mon[23733]: pgmap v149: 1 pgs: 1 remapped+peering; 577 KiB data, 40 MiB used, 626 GiB / 626 GiB avail 2024-09-21T04:06:59.265 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:06:58 smithi089 ceph-mon[23733]: osdmap e50: 7 total, 7 up, 7 in 2024-09-21T04:06:59.354 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:06:58 smithi195 ceph-mon[25840]: pgmap v149: 1 pgs: 1 remapped+peering; 577 KiB data, 40 MiB used, 626 GiB / 626 GiB avail 2024-09-21T04:06:59.354 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:06:58 smithi195 ceph-mon[25840]: osdmap e50: 7 total, 7 up, 7 in 2024-09-21T04:07:00.207 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:06:59 smithi195 ceph-mon[25840]: osdmap e51: 7 total, 7 up, 7 in 2024-09-21T04:07:00.208 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:06:59 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' 2024-09-21T04:07:00.208 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:06:59 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-21T04:07:00.208 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:06:59 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-21T04:07:00.208 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:06:59 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-21T04:07:00.208 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:06:59 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' 2024-09-21T04:07:00.208 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:06:59 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' 2024-09-21T04:07:00.208 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:06:59 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T04:07:00.208 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:06:59 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T04:07:00.208 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:06:59 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T04:07:00.208 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:06:59 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' 2024-09-21T04:07:00.265 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:06:59 smithi089 ceph-mon[23733]: osdmap e51: 7 total, 7 up, 7 in 2024-09-21T04:07:00.265 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:06:59 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' 2024-09-21T04:07:00.265 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:06:59 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-21T04:07:00.265 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:06:59 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-21T04:07:00.265 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:06:59 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-21T04:07:00.265 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:06:59 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' 2024-09-21T04:07:00.265 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:06:59 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' 2024-09-21T04:07:00.265 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:06:59 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T04:07:00.265 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:06:59 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T04:07:00.265 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:06:59 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T04:07:00.265 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:06:59 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' 2024-09-21T04:07:01.095 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:07:00 smithi195 ceph-mon[25840]: pgmap v152: 1 pgs: 1 remapped+peering; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-09-21T04:07:01.095 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:07:00 smithi195 ceph-mon[25840]: Detected new or changed devices on smithi195 2024-09-21T04:07:01.095 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:07:00 smithi195 ceph-mon[25840]: Adjusting osd_memory_target on smithi195 to 5312M 2024-09-21T04:07:01.095 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:07:00 smithi195 ceph-mon[25840]: osdmap e52: 7 total, 7 up, 7 in 2024-09-21T04:07:01.095 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:07:00 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-21T04:07:01.095 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:07:00 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-21T04:07:01.095 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:07:00 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T04:07:01.264 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:07:00 smithi089 ceph-mon[23733]: pgmap v152: 1 pgs: 1 remapped+peering; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-09-21T04:07:01.264 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:07:00 smithi089 ceph-mon[23733]: Detected new or changed devices on smithi195 2024-09-21T04:07:01.265 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:07:00 smithi089 ceph-mon[23733]: Adjusting osd_memory_target on smithi195 to 5312M 2024-09-21T04:07:01.265 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:07:00 smithi089 ceph-mon[23733]: osdmap e52: 7 total, 7 up, 7 in 2024-09-21T04:07:01.265 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:07:00 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-21T04:07:01.265 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:07:00 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-21T04:07:01.265 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:07:00 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T04:07:02.139 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:07:01 smithi195 ceph-mon[25840]: from='client.24201 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi195:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T04:07:02.264 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:07:01 smithi089 ceph-mon[23733]: from='client.24201 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi195:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T04:07:03.264 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:07:02 smithi089 ceph-mon[23733]: pgmap v154: 1 pgs: 1 remapped+peering; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-09-21T04:07:03.264 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:07:02 smithi089 ceph-mon[23733]: from='client.? 172.21.15.195:0/3579613401' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "50ea76bf-c840-4b8b-9910-dad0e4aafc23"}]: dispatch 2024-09-21T04:07:03.264 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:07:02 smithi089 ceph-mon[23733]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "50ea76bf-c840-4b8b-9910-dad0e4aafc23"}]: dispatch 2024-09-21T04:07:03.264 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:07:02 smithi089 ceph-mon[23733]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "50ea76bf-c840-4b8b-9910-dad0e4aafc23"}]': finished 2024-09-21T04:07:03.265 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:07:02 smithi089 ceph-mon[23733]: osdmap e53: 8 total, 7 up, 8 in 2024-09-21T04:07:03.265 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:07:02 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-21T04:07:03.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:07:02 smithi195 ceph-mon[25840]: pgmap v154: 1 pgs: 1 remapped+peering; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-09-21T04:07:03.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:07:02 smithi195 ceph-mon[25840]: from='client.? 172.21.15.195:0/3579613401' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "50ea76bf-c840-4b8b-9910-dad0e4aafc23"}]: dispatch 2024-09-21T04:07:03.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:07:02 smithi195 ceph-mon[25840]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "50ea76bf-c840-4b8b-9910-dad0e4aafc23"}]: dispatch 2024-09-21T04:07:03.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:07:02 smithi195 ceph-mon[25840]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "50ea76bf-c840-4b8b-9910-dad0e4aafc23"}]': finished 2024-09-21T04:07:03.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:07:02 smithi195 ceph-mon[25840]: osdmap e53: 8 total, 7 up, 8 in 2024-09-21T04:07:03.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:07:02 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-21T04:07:04.260 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:07:03 smithi195 ceph-mon[25840]: from='client.? 172.21.15.195:0/4153019607' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-21T04:07:04.264 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:07:03 smithi089 ceph-mon[23733]: from='client.? 172.21.15.195:0/4153019607' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-21T04:07:05.264 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:07:04 smithi089 ceph-mon[23733]: pgmap v156: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 100 KiB/s, 0 objects/s recovering 2024-09-21T04:07:05.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:07:04 smithi195 ceph-mon[25840]: pgmap v156: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 100 KiB/s, 0 objects/s recovering 2024-09-21T04:07:07.264 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:07:06 smithi089 ceph-mon[23733]: pgmap v157: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 84 KiB/s, 0 objects/s recovering 2024-09-21T04:07:07.281 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:07:06 smithi195 ceph-mon[25840]: pgmap v157: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 84 KiB/s, 0 objects/s recovering 2024-09-21T04:07:09.264 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:07:08 smithi089 ceph-mon[23733]: pgmap v158: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-09-21T04:07:09.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:07:08 smithi195 ceph-mon[25840]: pgmap v158: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-09-21T04:07:10.264 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:07:09 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-09-21T04:07:10.265 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:07:09 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T04:07:10.265 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:07:09 smithi089 ceph-mon[23733]: Deploying daemon osd.7 on smithi195 2024-09-21T04:07:10.294 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:07:09 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-09-21T04:07:10.294 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:07:09 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T04:07:10.295 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:07:09 smithi195 ceph-mon[25840]: Deploying daemon osd.7 on smithi195 2024-09-21T04:07:11.264 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:07:10 smithi089 ceph-mon[23733]: pgmap v159: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail; 56 KiB/s, 0 objects/s recovering 2024-09-21T04:07:11.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:07:10 smithi195 ceph-mon[25840]: pgmap v159: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail; 56 KiB/s, 0 objects/s recovering 2024-09-21T04:07:13.109 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:07:12 smithi195 ceph-mon[25840]: pgmap v160: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-09-21T04:07:13.264 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:07:12 smithi089 ceph-mon[23733]: pgmap v160: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-09-21T04:07:14.264 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:07:13 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi089.srlyxf/mirror_snapshot_schedule"}]: dispatch 2024-09-21T04:07:14.265 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:07:13 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi089.srlyxf/trash_purge_schedule"}]: dispatch 2024-09-21T04:07:14.265 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:07:13 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' 2024-09-21T04:07:14.318 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:07:13 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi089.srlyxf/mirror_snapshot_schedule"}]: dispatch 2024-09-21T04:07:14.318 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:07:13 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi089.srlyxf/trash_purge_schedule"}]: dispatch 2024-09-21T04:07:14.318 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:07:13 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' 2024-09-21T04:07:15.243 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:07:14 smithi195 ceph-mon[25840]: pgmap v161: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail 2024-09-21T04:07:15.243 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:07:14 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T04:07:15.243 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:07:14 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T04:07:15.244 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:07:14 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T04:07:15.264 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:07:14 smithi089 ceph-mon[23733]: pgmap v161: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail 2024-09-21T04:07:15.264 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:07:14 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T04:07:15.265 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:07:14 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T04:07:15.265 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:07:14 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T04:07:17.247 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:07:16 smithi195 ceph-mon[25840]: pgmap v162: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail 2024-09-21T04:07:17.264 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:07:16 smithi089 ceph-mon[23733]: pgmap v162: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail 2024-09-21T04:07:17.280 INFO:teuthology.orchestra.run.smithi195.stdout:Created osd(s) 7 on host 'smithi195' 2024-09-21T04:07:17.873 DEBUG:teuthology.orchestra.run.smithi195:osd.7> sudo journalctl -f -n 0 -u ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490@osd.7.service 2024-09-21T04:07:17.876 INFO:tasks.cephadm:Waiting for 8 OSDs to come up... 2024-09-21T04:07:17.876 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 1b9b79fc-77ce-11ef-baf5-efdc52797490 -- ceph osd stat -f json 2024-09-21T04:07:18.109 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:07:18 smithi195 ceph-mon[25840]: from='osd.7 [v2:172.21.15.195:6824/2424689581,v1:172.21.15.195:6825/2424689581]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-09-21T04:07:18.109 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:07:18 smithi195 ceph-mon[25840]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-09-21T04:07:18.109 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:07:18 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' 2024-09-21T04:07:18.182 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:07:18 smithi089 ceph-mon[23733]: from='osd.7 [v2:172.21.15.195:6824/2424689581,v1:172.21.15.195:6825/2424689581]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-09-21T04:07:18.182 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:07:18 smithi089 ceph-mon[23733]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-09-21T04:07:18.183 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:07:18 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' 2024-09-21T04:07:19.109 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:07:19 smithi195 ceph-mon[25840]: pgmap v163: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail 2024-09-21T04:07:19.109 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:07:19 smithi195 ceph-mon[25840]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-09-21T04:07:19.109 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:07:19 smithi195 ceph-mon[25840]: osdmap e54: 8 total, 7 up, 8 in 2024-09-21T04:07:19.109 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:07:19 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-21T04:07:19.109 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:07:19 smithi195 ceph-mon[25840]: from='osd.7 [v2:172.21.15.195:6824/2424689581,v1:172.21.15.195:6825/2424689581]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi195", "root=default"]}]: dispatch 2024-09-21T04:07:19.109 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:07:19 smithi195 ceph-mon[25840]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi195", "root=default"]}]: dispatch 2024-09-21T04:07:19.109 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:07:19 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' 2024-09-21T04:07:19.109 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 21 04:07:19 smithi195 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-7[39020]: 2024-09-21T04:07:19.011+0000 7ffa7b0d4700 -1 osd.7 0 waiting for initial osdmap 2024-09-21T04:07:19.110 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 21 04:07:19 smithi195 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-7[39020]: 2024-09-21T04:07:19.019+0000 7ffa75a6b700 -1 osd.7 55 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-21T04:07:19.264 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:07:19 smithi089 ceph-mon[23733]: pgmap v163: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail 2024-09-21T04:07:19.264 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:07:19 smithi089 ceph-mon[23733]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-09-21T04:07:19.265 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:07:19 smithi089 ceph-mon[23733]: osdmap e54: 8 total, 7 up, 8 in 2024-09-21T04:07:19.265 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:07:19 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-21T04:07:19.265 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:07:19 smithi089 ceph-mon[23733]: from='osd.7 [v2:172.21.15.195:6824/2424689581,v1:172.21.15.195:6825/2424689581]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi195", "root=default"]}]: dispatch 2024-09-21T04:07:19.265 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:07:19 smithi089 ceph-mon[23733]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi195", "root=default"]}]: dispatch 2024-09-21T04:07:19.265 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:07:19 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' 2024-09-21T04:07:19.929 INFO:teuthology.orchestra.run.smithi089.stdout: 2024-09-21T04:07:20.264 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:07:20 smithi089 ceph-mon[23733]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi195", "root=default"]}]': finished 2024-09-21T04:07:20.265 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:07:20 smithi089 ceph-mon[23733]: osdmap e55: 8 total, 7 up, 8 in 2024-09-21T04:07:20.265 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:07:20 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-21T04:07:20.265 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:07:20 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-21T04:07:20.265 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:07:20 smithi089 ceph-mon[23733]: from='client.? 172.21.15.89:0/3856859085' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-09-21T04:07:20.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:07:20 smithi195 ceph-mon[25840]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi195", "root=default"]}]': finished 2024-09-21T04:07:20.783 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:07:20 smithi195 ceph-mon[25840]: osdmap e55: 8 total, 7 up, 8 in 2024-09-21T04:07:20.783 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:07:20 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-21T04:07:20.783 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:07:20 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-21T04:07:20.783 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:07:20 smithi195 ceph-mon[25840]: from='client.? 172.21.15.89:0/3856859085' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-09-21T04:07:20.784 INFO:teuthology.orchestra.run.smithi089.stdout:{"epoch":55,"num_osds":8,"num_up_osds":7,"osd_up_since":1726891615,"num_in_osds":8,"osd_in_since":1726891622,"num_remapped_pgs":0} 2024-09-21T04:07:21.109 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:07:21 smithi195 ceph-mon[25840]: purged_snaps scrub starts 2024-09-21T04:07:21.109 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:07:21 smithi195 ceph-mon[25840]: purged_snaps scrub ok 2024-09-21T04:07:21.109 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:07:21 smithi195 ceph-mon[25840]: pgmap v166: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail 2024-09-21T04:07:21.109 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:07:21 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-21T04:07:21.109 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:07:21 smithi195 ceph-mon[25840]: osd.7 [v2:172.21.15.195:6824/2424689581,v1:172.21.15.195:6825/2424689581] boot 2024-09-21T04:07:21.109 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:07:21 smithi195 ceph-mon[25840]: osdmap e56: 8 total, 8 up, 8 in 2024-09-21T04:07:21.109 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:07:21 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-21T04:07:21.264 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:07:21 smithi089 ceph-mon[23733]: purged_snaps scrub starts 2024-09-21T04:07:21.265 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:07:21 smithi089 ceph-mon[23733]: purged_snaps scrub ok 2024-09-21T04:07:21.265 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:07:21 smithi089 ceph-mon[23733]: pgmap v166: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail 2024-09-21T04:07:21.265 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:07:21 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-21T04:07:21.265 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:07:21 smithi089 ceph-mon[23733]: osd.7 [v2:172.21.15.195:6824/2424689581,v1:172.21.15.195:6825/2424689581] boot 2024-09-21T04:07:21.265 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:07:21 smithi089 ceph-mon[23733]: osdmap e56: 8 total, 8 up, 8 in 2024-09-21T04:07:21.265 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:07:21 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-21T04:07:21.785 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 1b9b79fc-77ce-11ef-baf5-efdc52797490 -- ceph osd stat -f json 2024-09-21T04:07:22.025 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:07:22 smithi089 ceph-mon[23733]: osdmap e57: 8 total, 8 up, 8 in 2024-09-21T04:07:22.025 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:07:22 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' 2024-09-21T04:07:22.025 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:07:22 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-21T04:07:22.025 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:07:22 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-21T04:07:22.025 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:07:22 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-21T04:07:22.025 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:07:22 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-21T04:07:22.025 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:07:22 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' 2024-09-21T04:07:22.025 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:07:22 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' 2024-09-21T04:07:22.025 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:07:22 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T04:07:22.025 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:07:22 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T04:07:22.025 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:07:22 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T04:07:22.026 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:07:22 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' 2024-09-21T04:07:22.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:07:22 smithi195 ceph-mon[25840]: osdmap e57: 8 total, 8 up, 8 in 2024-09-21T04:07:22.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:07:22 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' 2024-09-21T04:07:22.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:07:22 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-21T04:07:22.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:07:22 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-21T04:07:22.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:07:22 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-21T04:07:22.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:07:22 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-21T04:07:22.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:07:22 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' 2024-09-21T04:07:22.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:07:22 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' 2024-09-21T04:07:22.360 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:07:22 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T04:07:22.360 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:07:22 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T04:07:22.360 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:07:22 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T04:07:22.360 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:07:22 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' 2024-09-21T04:07:23.128 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:07:23 smithi089 ceph-mon[23733]: pgmap v169: 1 pgs: 1 remapped+peering; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2024-09-21T04:07:23.128 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:07:23 smithi089 ceph-mon[23733]: Detected new or changed devices on smithi195 2024-09-21T04:07:23.128 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:07:23 smithi089 ceph-mon[23733]: Adjusting osd_memory_target on smithi195 to 3984M 2024-09-21T04:07:23.128 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:07:23 smithi089 ceph-mon[23733]: osdmap e58: 8 total, 8 up, 8 in 2024-09-21T04:07:23.358 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:07:23 smithi195 ceph-mon[25840]: pgmap v169: 1 pgs: 1 remapped+peering; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2024-09-21T04:07:23.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:07:23 smithi195 ceph-mon[25840]: Detected new or changed devices on smithi195 2024-09-21T04:07:23.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:07:23 smithi195 ceph-mon[25840]: Adjusting osd_memory_target on smithi195 to 3984M 2024-09-21T04:07:23.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:07:23 smithi195 ceph-mon[25840]: osdmap e58: 8 total, 8 up, 8 in 2024-09-21T04:07:23.702 INFO:teuthology.orchestra.run.smithi089.stdout: 2024-09-21T04:07:24.351 INFO:teuthology.orchestra.run.smithi089.stdout:{"epoch":59,"num_osds":8,"num_up_osds":8,"osd_up_since":1726891640,"num_in_osds":8,"osd_in_since":1726891622,"num_remapped_pgs":0} 2024-09-21T04:07:24.352 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 1b9b79fc-77ce-11ef-baf5-efdc52797490 -- ceph osd dump --format=json 2024-09-21T04:07:24.375 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:07:24 smithi089 ceph-mon[23733]: pgmap v171: 1 pgs: 1 remapped+peering; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2024-09-21T04:07:24.375 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:07:24 smithi089 ceph-mon[23733]: osdmap e59: 8 total, 8 up, 8 in 2024-09-21T04:07:24.375 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:07:24 smithi089 ceph-mon[23733]: from='client.? 172.21.15.89:0/3134660358' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-09-21T04:07:24.557 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/1b9b79fc-77ce-11ef-baf5-efdc52797490/mon.smithi089/config 2024-09-21T04:07:24.608 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:07:24 smithi195 ceph-mon[25840]: pgmap v171: 1 pgs: 1 remapped+peering; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2024-09-21T04:07:24.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:07:24 smithi195 ceph-mon[25840]: osdmap e59: 8 total, 8 up, 8 in 2024-09-21T04:07:24.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:07:24 smithi195 ceph-mon[25840]: from='client.? 172.21.15.89:0/3134660358' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-09-21T04:07:25.370 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:07:25 smithi089 ceph-mon[23733]: osdmap e60: 8 total, 8 up, 8 in 2024-09-21T04:07:25.608 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:07:25 smithi195 ceph-mon[25840]: osdmap e60: 8 total, 8 up, 8 in 2024-09-21T04:07:26.363 INFO:teuthology.orchestra.run.smithi089.stdout: 2024-09-21T04:07:26.364 INFO:teuthology.orchestra.run.smithi089.stdout:{"epoch":60,"fsid":"1b9b79fc-77ce-11ef-baf5-efdc52797490","created":"2024-09-21T04:01:32.259098+0000","modified":"2024-09-21T04:07:24.113904+0000","last_up_change":"2024-09-21T04:07:20.009578+0000","last_in_change":"2024-09-21T04:07:02.878002+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-21T04:05:19.372244+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"23","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}}],"osds":[{"osd":0,"uuid":"cb916820-cc81-471e-82a5-0ac43bae0af0","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":58,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6802","nonce":2758701754},{"type":"v1","addr":"172.21.15.89:6803","nonce":2758701754}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6804","nonce":2758701754},{"type":"v1","addr":"172.21.15.89:6805","nonce":2758701754}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6808","nonce":2758701754},{"type":"v1","addr":"172.21.15.89:6809","nonce":2758701754}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6806","nonce":2758701754},{"type":"v1","addr":"172.21.15.89:6807","nonce":2758701754}]},"public_addr":"172.21.15.89:6803/2758701754","cluster_addr":"172.21.15.89:6805/2758701754","heartbeat_back_addr":"172.21.15.89:6809/2758701754","heartbeat_front_addr":"172.21.15.89:6807/2758701754","state":["exists","up"]},{"osd":1,"uuid":"3e63078d-befa-41e7-b13f-11fd3a63588a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":35,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6810","nonce":2620086826},{"type":"v1","addr":"172.21.15.89:6811","nonce":2620086826}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6812","nonce":2620086826},{"type":"v1","addr":"172.21.15.89:6813","nonce":2620086826}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6816","nonce":2620086826},{"type":"v1","addr":"172.21.15.89:6817","nonce":2620086826}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6814","nonce":2620086826},{"type":"v1","addr":"172.21.15.89:6815","nonce":2620086826}]},"public_addr":"172.21.15.89:6811/2620086826","cluster_addr":"172.21.15.89:6813/2620086826","heartbeat_back_addr":"172.21.15.89:6817/2620086826","heartbeat_front_addr":"172.21.15.89:6815/2620086826","state":["exists","up"]},{"osd":2,"uuid":"99ec6c1f-c76e-4f79-b030-b980e297e1cb","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":19,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6818","nonce":3898997901},{"type":"v1","addr":"172.21.15.89:6819","nonce":3898997901}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6820","nonce":3898997901},{"type":"v1","addr":"172.21.15.89:6821","nonce":3898997901}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6824","nonce":3898997901},{"type":"v1","addr":"172.21.15.89:6825","nonce":3898997901}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6822","nonce":3898997901},{"type":"v1","addr":"172.21.15.89:6823","nonce":3898997901}]},"public_addr":"172.21.15.89:6819/3898997901","cluster_addr":"172.21.15.89:6821/3898997901","heartbeat_back_addr":"172.21.15.89:6825/3898997901","heartbeat_front_addr":"172.21.15.89:6823/3898997901","state":["exists","up"]},{"osd":3,"uuid":"288ca940-ca66-41f1-894f-32ac0c9d9eb3","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":27,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6826","nonce":2601107822},{"type":"v1","addr":"172.21.15.89:6827","nonce":2601107822}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6828","nonce":2601107822},{"type":"v1","addr":"172.21.15.89:6829","nonce":2601107822}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6832","nonce":2601107822},{"type":"v1","addr":"172.21.15.89:6833","nonce":2601107822}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6830","nonce":2601107822},{"type":"v1","addr":"172.21.15.89:6831","nonce":2601107822}]},"public_addr":"172.21.15.89:6827/2601107822","cluster_addr":"172.21.15.89:6829/2601107822","heartbeat_back_addr":"172.21.15.89:6833/2601107822","heartbeat_front_addr":"172.21.15.89:6831/2601107822","state":["exists","up"]},{"osd":4,"uuid":"b35557d2-875d-420b-ac49-a0cfe914822d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":32,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6800","nonce":44100003},{"type":"v1","addr":"172.21.15.195:6801","nonce":44100003}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6802","nonce":44100003},{"type":"v1","addr":"172.21.15.195:6803","nonce":44100003}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6806","nonce":44100003},{"type":"v1","addr":"172.21.15.195:6807","nonce":44100003}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6804","nonce":44100003},{"type":"v1","addr":"172.21.15.195:6805","nonce":44100003}]},"public_addr":"172.21.15.195:6801/44100003","cluster_addr":"172.21.15.195:6803/44100003","heartbeat_back_addr":"172.21.15.195:6807/44100003","heartbeat_front_addr":"172.21.15.195:6805/44100003","state":["exists","up"]},{"osd":5,"uuid":"94ffbc65-1d43-460b-84eb-6be843a72bae","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":40,"up_thru":43,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6808","nonce":1913588562},{"type":"v1","addr":"172.21.15.195:6809","nonce":1913588562}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6810","nonce":1913588562},{"type":"v1","addr":"172.21.15.195:6811","nonce":1913588562}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6814","nonce":1913588562},{"type":"v1","addr":"172.21.15.195:6815","nonce":1913588562}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6812","nonce":1913588562},{"type":"v1","addr":"172.21.15.195:6813","nonce":1913588562}]},"public_addr":"172.21.15.195:6809/1913588562","cluster_addr":"172.21.15.195:6811/1913588562","heartbeat_back_addr":"172.21.15.195:6815/1913588562","heartbeat_front_addr":"172.21.15.195:6813/1913588562","state":["exists","up"]},{"osd":6,"uuid":"3ae73a35-8e39-47b1-8f1b-e52c81bbf9c8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":48,"up_thru":51,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6816","nonce":1620163911},{"type":"v1","addr":"172.21.15.195:6817","nonce":1620163911}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6818","nonce":1620163911},{"type":"v1","addr":"172.21.15.195:6819","nonce":1620163911}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6822","nonce":1620163911},{"type":"v1","addr":"172.21.15.195:6823","nonce":1620163911}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6820","nonce":1620163911},{"type":"v1","addr":"172.21.15.195:6821","nonce":1620163911}]},"public_addr":"172.21.15.195:6817/1620163911","cluster_addr":"172.21.15.195:6819/1620163911","heartbeat_back_addr":"172.21.15.195:6823/1620163911","heartbeat_front_addr":"172.21.15.195:6821/1620163911","state":["exists","up"]},{"osd":7,"uuid":"50ea76bf-c840-4b8b-9910-dad0e4aafc23","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":56,"up_thru":59,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6824","nonce":2424689581},{"type":"v1","addr":"172.21.15.195:6825","nonce":2424689581}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6826","nonce":2424689581},{"type":"v1","addr":"172.21.15.195:6827","nonce":2424689581}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6830","nonce":2424689581},{"type":"v1","addr":"172.21.15.195:6831","nonce":2424689581}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6828","nonce":2424689581},{"type":"v1","addr":"172.21.15.195:6829","nonce":2424689581}]},"public_addr":"172.21.15.195:6825/2424689581","cluster_addr":"172.21.15.195:6827/2424689581","heartbeat_back_addr":"172.21.15.195:6831/2424689581","heartbeat_front_addr":"172.21.15.195:6829/2424689581","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T04:04:18.651040+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T04:04:48.404481+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T04:05:16.848563+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T04:05:44.741921+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T04:06:06.786033+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T04:06:30.128500+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T04:06:54.330921+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T04:07:18.049029+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.89:6801/1123747776":"2024-09-22T04:03:13.037420+0000","172.21.15.89:6800/1123747776":"2024-09-22T04:03:13.037420+0000","172.21.15.89:0/3294782937":"2024-09-22T04:02:24.242528+0000","172.21.15.89:0/2709663841":"2024-09-22T04:02:24.242528+0000","172.21.15.89:6801/1983486000":"2024-09-22T04:02:24.242528+0000","172.21.15.89:6800/1983486000":"2024-09-22T04:02:24.242528+0000","172.21.15.89:0/3099511971":"2024-09-22T04:01:55.211457+0000","172.21.15.89:0/328064113":"2024-09-22T04:03:13.037420+0000","172.21.15.89:0/2787795518":"2024-09-22T04:02:24.242528+0000","172.21.15.89:6800/4163677862":"2024-09-22T04:01:55.211457+0000","172.21.15.89:0/2972904652":"2024-09-22T04:01:55.211457+0000","172.21.15.89:0/3651427575":"2024-09-22T04:03:13.037420+0000","172.21.15.89:0/2159750059":"2024-09-22T04:03:13.037420+0000","172.21.15.89:6801/4163677862":"2024-09-22T04:01:55.211457+0000","172.21.15.89:0/2242250733":"2024-09-22T04:01:55.211457+0000"},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-21T04:07:26.364 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:07:26 smithi089 ceph-mon[23733]: pgmap v174: 1 pgs: 1 remapped+peering; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2024-09-21T04:07:26.608 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:07:26 smithi195 ceph-mon[25840]: pgmap v174: 1 pgs: 1 remapped+peering; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2024-09-21T04:07:27.053 INFO:tasks.cephadm.ceph_manager.ceph:[{'pool': 1, 'pool_name': '.mgr', 'create_time': '2024-09-21T04:05:19.372244+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '23', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}}] 2024-09-21T04:07:27.053 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 1b9b79fc-77ce-11ef-baf5-efdc52797490 -- ceph osd pool get .mgr pg_num 2024-09-21T04:07:27.298 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/1b9b79fc-77ce-11ef-baf5-efdc52797490/mon.smithi089/config 2024-09-21T04:07:27.414 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:07:27 smithi089 ceph-mon[23733]: from='client.? 172.21.15.89:0/563118960' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-21T04:07:27.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:07:27 smithi195 ceph-mon[25840]: from='client.? 172.21.15.89:0/563118960' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-21T04:07:28.396 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:07:28 smithi089 ceph-mon[23733]: pgmap v175: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-09-21T04:07:28.608 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:07:28 smithi195 ceph-mon[25840]: pgmap v175: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-09-21T04:07:29.025 INFO:teuthology.orchestra.run.smithi089.stdout:pg_num: 1 2024-09-21T04:07:29.514 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:07:29 smithi089 ceph-mon[23733]: from='client.? 172.21.15.89:0/2242092676' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-09-21T04:07:29.608 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:07:29 smithi195 ceph-mon[25840]: from='client.? 172.21.15.89:0/2242092676' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-09-21T04:07:29.769 INFO:tasks.cephadm:Setting up client nodes... 2024-09-21T04:07:29.769 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 1b9b79fc-77ce-11ef-baf5-efdc52797490 -- ceph auth get-or-create client.0 mon 'allow *' osd 'allow *' mds 'allow *' mgr 'allow *' 2024-09-21T04:07:30.382 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:07:30 smithi089 ceph-mon[23733]: pgmap v176: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 73 KiB/s, 0 objects/s recovering 2024-09-21T04:07:30.595 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:07:30 smithi195 ceph-mon[25840]: pgmap v176: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 73 KiB/s, 0 objects/s recovering 2024-09-21T04:07:31.784 INFO:teuthology.orchestra.run.smithi089.stdout:[client.0] 2024-09-21T04:07:31.784 INFO:teuthology.orchestra.run.smithi089.stdout: key = AQCDRu5mwhVnLhAAuulQISf9XXsZijlcJZfXCA== 2024-09-21T04:07:32.445 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:07:32 smithi089 ceph-mon[23733]: pgmap v177: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-09-21T04:07:32.445 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:07:32 smithi089 ceph-mon[23733]: from='client.? 172.21.15.89:0/1575033474' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-09-21T04:07:32.445 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:07:32 smithi089 ceph-mon[23733]: from='client.? 172.21.15.89:0/1575033474' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-09-21T04:07:32.467 DEBUG:teuthology.orchestra.run.smithi089:> set -ex 2024-09-21T04:07:32.468 DEBUG:teuthology.orchestra.run.smithi089:> sudo dd of=/etc/ceph/ceph.client.0.keyring 2024-09-21T04:07:32.468 DEBUG:teuthology.orchestra.run.smithi089:> sudo chmod 0644 /etc/ceph/ceph.client.0.keyring 2024-09-21T04:07:32.504 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 1b9b79fc-77ce-11ef-baf5-efdc52797490 -- ceph config log 1 --format=json 2024-09-21T04:07:32.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:07:32 smithi195 ceph-mon[25840]: pgmap v177: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-09-21T04:07:32.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:07:32 smithi195 ceph-mon[25840]: from='client.? 172.21.15.89:0/1575033474' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-09-21T04:07:32.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:07:32 smithi195 ceph-mon[25840]: from='client.? 172.21.15.89:0/1575033474' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-09-21T04:07:32.746 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/1b9b79fc-77ce-11ef-baf5-efdc52797490/mon.smithi089/config 2024-09-21T04:07:34.499 INFO:teuthology.orchestra.run.smithi089.stdout: 2024-09-21T04:07:34.499 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:07:34 smithi089 ceph-mon[23733]: pgmap v178: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-09-21T04:07:34.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:07:34 smithi195 ceph-mon[25840]: pgmap v178: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-09-21T04:07:35.280 INFO:teuthology.orchestra.run.smithi089.stdout:[{"version":20,"timestamp":"2024-09-21T04:07:21.466664+0000","name":"","changes":[{"name":"osd/host:smithi195/osd_memory_target","previous_value":"5570054007","new_value":"4177540505"}]}] 2024-09-21T04:07:35.280 INFO:tasks.ceph_manager:config epoch is 20 2024-09-21T04:07:35.280 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2024-09-21T04:07:35.280 INFO:tasks.cephadm.ceph_manager.ceph:waiting for mgr available 2024-09-21T04:07:35.280 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 1b9b79fc-77ce-11ef-baf5-efdc52797490 -- ceph mgr dump --format=json 2024-09-21T04:07:35.305 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:07:35 smithi089 ceph-mon[23733]: from='client.? 172.21.15.89:0/896499001' entity='client.admin' cmd=[{"prefix": "config log", "num": 1, "format": "json"}]: dispatch 2024-09-21T04:07:35.358 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:07:35 smithi195 ceph-mon[25840]: from='client.? 172.21.15.89:0/896499001' entity='client.admin' cmd=[{"prefix": "config log", "num": 1, "format": "json"}]: dispatch 2024-09-21T04:07:35.489 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/1b9b79fc-77ce-11ef-baf5-efdc52797490/mon.smithi089/config 2024-09-21T04:07:36.264 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:07:36 smithi089 ceph-mon[23733]: pgmap v179: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 47 KiB/s, 0 objects/s recovering 2024-09-21T04:07:36.608 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:07:36 smithi195 ceph-mon[25840]: pgmap v179: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 47 KiB/s, 0 objects/s recovering 2024-09-21T04:07:37.364 INFO:teuthology.orchestra.run.smithi089.stdout: 2024-09-21T04:07:38.033 INFO:teuthology.orchestra.run.smithi089.stdout:{"epoch":19,"active_gid":14200,"active_name":"smithi089.srlyxf","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6800","nonce":717134025},{"type":"v1","addr":"172.21.15.89:6801","nonce":717134025}]},"active_addr":"172.21.15.89:6801/717134025","active_change":"2024-09-21T04:03:13.037690+0000","active_mgr_features":4540138303579357183,"available":true,"standbys":[{"gid":14220,"name":"smithi195.cgotzm","mgr_features":4540138303579357183,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.23.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:8.3.5","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"docker.io/library/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"docker.io/arcts/keepalived","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.3.1","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.33.4","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.23.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:8.3.5","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"docker.io/library/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"docker.io/arcts/keepalived","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.3.1","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.33.4","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{"dashboard":"https://172.21.15.89:8443/","prometheus":"http://172.21.15.89:9283/"},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"last_failure_osd_epoch":5,"active_clients":[{"addrvec":[{"type":"v2","addr":"172.21.15.89:0","nonce":1389126175}]},{"addrvec":[{"type":"v2","addr":"172.21.15.89:0","nonce":1622894153}]},{"addrvec":[{"type":"v2","addr":"172.21.15.89:0","nonce":1717078521}]},{"addrvec":[{"type":"v2","addr":"172.21.15.89:0","nonce":349445506}]}]}} 2024-09-21T04:07:38.036 INFO:tasks.cephadm.ceph_manager.ceph:mgr available! 2024-09-21T04:07:38.036 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2024-09-21T04:07:38.036 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 1b9b79fc-77ce-11ef-baf5-efdc52797490 -- ceph osd dump --format=json 2024-09-21T04:07:38.264 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:07:38 smithi089 ceph-mon[23733]: pgmap v180: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 43 KiB/s, 0 objects/s recovering 2024-09-21T04:07:38.264 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:07:38 smithi089 ceph-mon[23733]: from='client.? 172.21.15.89:0/812535384' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-09-21T04:07:38.280 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/1b9b79fc-77ce-11ef-baf5-efdc52797490/mon.smithi089/config 2024-09-21T04:07:38.608 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:07:38 smithi195 ceph-mon[25840]: pgmap v180: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 43 KiB/s, 0 objects/s recovering 2024-09-21T04:07:38.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:07:38 smithi195 ceph-mon[25840]: from='client.? 172.21.15.89:0/812535384' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-09-21T04:07:40.047 INFO:teuthology.orchestra.run.smithi089.stdout: 2024-09-21T04:07:40.048 INFO:teuthology.orchestra.run.smithi089.stdout:{"epoch":60,"fsid":"1b9b79fc-77ce-11ef-baf5-efdc52797490","created":"2024-09-21T04:01:32.259098+0000","modified":"2024-09-21T04:07:24.113904+0000","last_up_change":"2024-09-21T04:07:20.009578+0000","last_in_change":"2024-09-21T04:07:02.878002+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-21T04:05:19.372244+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"23","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}}],"osds":[{"osd":0,"uuid":"cb916820-cc81-471e-82a5-0ac43bae0af0","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":58,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6802","nonce":2758701754},{"type":"v1","addr":"172.21.15.89:6803","nonce":2758701754}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6804","nonce":2758701754},{"type":"v1","addr":"172.21.15.89:6805","nonce":2758701754}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6808","nonce":2758701754},{"type":"v1","addr":"172.21.15.89:6809","nonce":2758701754}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6806","nonce":2758701754},{"type":"v1","addr":"172.21.15.89:6807","nonce":2758701754}]},"public_addr":"172.21.15.89:6803/2758701754","cluster_addr":"172.21.15.89:6805/2758701754","heartbeat_back_addr":"172.21.15.89:6809/2758701754","heartbeat_front_addr":"172.21.15.89:6807/2758701754","state":["exists","up"]},{"osd":1,"uuid":"3e63078d-befa-41e7-b13f-11fd3a63588a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":35,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6810","nonce":2620086826},{"type":"v1","addr":"172.21.15.89:6811","nonce":2620086826}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6812","nonce":2620086826},{"type":"v1","addr":"172.21.15.89:6813","nonce":2620086826}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6816","nonce":2620086826},{"type":"v1","addr":"172.21.15.89:6817","nonce":2620086826}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6814","nonce":2620086826},{"type":"v1","addr":"172.21.15.89:6815","nonce":2620086826}]},"public_addr":"172.21.15.89:6811/2620086826","cluster_addr":"172.21.15.89:6813/2620086826","heartbeat_back_addr":"172.21.15.89:6817/2620086826","heartbeat_front_addr":"172.21.15.89:6815/2620086826","state":["exists","up"]},{"osd":2,"uuid":"99ec6c1f-c76e-4f79-b030-b980e297e1cb","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":19,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6818","nonce":3898997901},{"type":"v1","addr":"172.21.15.89:6819","nonce":3898997901}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6820","nonce":3898997901},{"type":"v1","addr":"172.21.15.89:6821","nonce":3898997901}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6824","nonce":3898997901},{"type":"v1","addr":"172.21.15.89:6825","nonce":3898997901}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6822","nonce":3898997901},{"type":"v1","addr":"172.21.15.89:6823","nonce":3898997901}]},"public_addr":"172.21.15.89:6819/3898997901","cluster_addr":"172.21.15.89:6821/3898997901","heartbeat_back_addr":"172.21.15.89:6825/3898997901","heartbeat_front_addr":"172.21.15.89:6823/3898997901","state":["exists","up"]},{"osd":3,"uuid":"288ca940-ca66-41f1-894f-32ac0c9d9eb3","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":27,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6826","nonce":2601107822},{"type":"v1","addr":"172.21.15.89:6827","nonce":2601107822}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6828","nonce":2601107822},{"type":"v1","addr":"172.21.15.89:6829","nonce":2601107822}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6832","nonce":2601107822},{"type":"v1","addr":"172.21.15.89:6833","nonce":2601107822}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6830","nonce":2601107822},{"type":"v1","addr":"172.21.15.89:6831","nonce":2601107822}]},"public_addr":"172.21.15.89:6827/2601107822","cluster_addr":"172.21.15.89:6829/2601107822","heartbeat_back_addr":"172.21.15.89:6833/2601107822","heartbeat_front_addr":"172.21.15.89:6831/2601107822","state":["exists","up"]},{"osd":4,"uuid":"b35557d2-875d-420b-ac49-a0cfe914822d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":32,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6800","nonce":44100003},{"type":"v1","addr":"172.21.15.195:6801","nonce":44100003}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6802","nonce":44100003},{"type":"v1","addr":"172.21.15.195:6803","nonce":44100003}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6806","nonce":44100003},{"type":"v1","addr":"172.21.15.195:6807","nonce":44100003}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6804","nonce":44100003},{"type":"v1","addr":"172.21.15.195:6805","nonce":44100003}]},"public_addr":"172.21.15.195:6801/44100003","cluster_addr":"172.21.15.195:6803/44100003","heartbeat_back_addr":"172.21.15.195:6807/44100003","heartbeat_front_addr":"172.21.15.195:6805/44100003","state":["exists","up"]},{"osd":5,"uuid":"94ffbc65-1d43-460b-84eb-6be843a72bae","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":40,"up_thru":43,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6808","nonce":1913588562},{"type":"v1","addr":"172.21.15.195:6809","nonce":1913588562}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6810","nonce":1913588562},{"type":"v1","addr":"172.21.15.195:6811","nonce":1913588562}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6814","nonce":1913588562},{"type":"v1","addr":"172.21.15.195:6815","nonce":1913588562}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6812","nonce":1913588562},{"type":"v1","addr":"172.21.15.195:6813","nonce":1913588562}]},"public_addr":"172.21.15.195:6809/1913588562","cluster_addr":"172.21.15.195:6811/1913588562","heartbeat_back_addr":"172.21.15.195:6815/1913588562","heartbeat_front_addr":"172.21.15.195:6813/1913588562","state":["exists","up"]},{"osd":6,"uuid":"3ae73a35-8e39-47b1-8f1b-e52c81bbf9c8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":48,"up_thru":51,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6816","nonce":1620163911},{"type":"v1","addr":"172.21.15.195:6817","nonce":1620163911}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6818","nonce":1620163911},{"type":"v1","addr":"172.21.15.195:6819","nonce":1620163911}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6822","nonce":1620163911},{"type":"v1","addr":"172.21.15.195:6823","nonce":1620163911}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6820","nonce":1620163911},{"type":"v1","addr":"172.21.15.195:6821","nonce":1620163911}]},"public_addr":"172.21.15.195:6817/1620163911","cluster_addr":"172.21.15.195:6819/1620163911","heartbeat_back_addr":"172.21.15.195:6823/1620163911","heartbeat_front_addr":"172.21.15.195:6821/1620163911","state":["exists","up"]},{"osd":7,"uuid":"50ea76bf-c840-4b8b-9910-dad0e4aafc23","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":56,"up_thru":59,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6824","nonce":2424689581},{"type":"v1","addr":"172.21.15.195:6825","nonce":2424689581}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6826","nonce":2424689581},{"type":"v1","addr":"172.21.15.195:6827","nonce":2424689581}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6830","nonce":2424689581},{"type":"v1","addr":"172.21.15.195:6831","nonce":2424689581}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6828","nonce":2424689581},{"type":"v1","addr":"172.21.15.195:6829","nonce":2424689581}]},"public_addr":"172.21.15.195:6825/2424689581","cluster_addr":"172.21.15.195:6827/2424689581","heartbeat_back_addr":"172.21.15.195:6831/2424689581","heartbeat_front_addr":"172.21.15.195:6829/2424689581","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T04:04:18.651040+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T04:04:48.404481+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T04:05:16.848563+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T04:05:44.741921+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T04:06:06.786033+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T04:06:30.128500+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T04:06:54.330921+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T04:07:18.049029+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.89:6801/1123747776":"2024-09-22T04:03:13.037420+0000","172.21.15.89:6800/1123747776":"2024-09-22T04:03:13.037420+0000","172.21.15.89:0/3294782937":"2024-09-22T04:02:24.242528+0000","172.21.15.89:0/2709663841":"2024-09-22T04:02:24.242528+0000","172.21.15.89:6801/1983486000":"2024-09-22T04:02:24.242528+0000","172.21.15.89:6800/1983486000":"2024-09-22T04:02:24.242528+0000","172.21.15.89:0/3099511971":"2024-09-22T04:01:55.211457+0000","172.21.15.89:0/328064113":"2024-09-22T04:03:13.037420+0000","172.21.15.89:0/2787795518":"2024-09-22T04:02:24.242528+0000","172.21.15.89:6800/4163677862":"2024-09-22T04:01:55.211457+0000","172.21.15.89:0/2972904652":"2024-09-22T04:01:55.211457+0000","172.21.15.89:0/3651427575":"2024-09-22T04:03:13.037420+0000","172.21.15.89:0/2159750059":"2024-09-22T04:03:13.037420+0000","172.21.15.89:6801/4163677862":"2024-09-22T04:01:55.211457+0000","172.21.15.89:0/2242250733":"2024-09-22T04:01:55.211457+0000"},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-21T04:07:40.514 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:07:40 smithi089 ceph-mon[23733]: pgmap v181: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-21T04:07:40.515 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:07:40 smithi089 ceph-mon[23733]: from='client.? 172.21.15.89:0/1625657586' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-21T04:07:40.595 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:07:40 smithi195 ceph-mon[25840]: pgmap v181: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-21T04:07:40.596 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:07:40 smithi195 ceph-mon[25840]: from='client.? 172.21.15.89:0/1625657586' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-21T04:07:40.664 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2024-09-21T04:07:40.665 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 1b9b79fc-77ce-11ef-baf5-efdc52797490 -- ceph osd dump --format=json 2024-09-21T04:07:40.869 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/1b9b79fc-77ce-11ef-baf5-efdc52797490/mon.smithi089/config 2024-09-21T04:07:42.354 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:07:42 smithi089 ceph-mon[23733]: pgmap v182: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-21T04:07:42.608 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:07:42 smithi195 ceph-mon[25840]: pgmap v182: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-21T04:07:42.720 INFO:teuthology.orchestra.run.smithi089.stdout: 2024-09-21T04:07:42.720 INFO:teuthology.orchestra.run.smithi089.stdout:{"epoch":60,"fsid":"1b9b79fc-77ce-11ef-baf5-efdc52797490","created":"2024-09-21T04:01:32.259098+0000","modified":"2024-09-21T04:07:24.113904+0000","last_up_change":"2024-09-21T04:07:20.009578+0000","last_in_change":"2024-09-21T04:07:02.878002+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-21T04:05:19.372244+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"23","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}}],"osds":[{"osd":0,"uuid":"cb916820-cc81-471e-82a5-0ac43bae0af0","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":58,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6802","nonce":2758701754},{"type":"v1","addr":"172.21.15.89:6803","nonce":2758701754}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6804","nonce":2758701754},{"type":"v1","addr":"172.21.15.89:6805","nonce":2758701754}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6808","nonce":2758701754},{"type":"v1","addr":"172.21.15.89:6809","nonce":2758701754}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6806","nonce":2758701754},{"type":"v1","addr":"172.21.15.89:6807","nonce":2758701754}]},"public_addr":"172.21.15.89:6803/2758701754","cluster_addr":"172.21.15.89:6805/2758701754","heartbeat_back_addr":"172.21.15.89:6809/2758701754","heartbeat_front_addr":"172.21.15.89:6807/2758701754","state":["exists","up"]},{"osd":1,"uuid":"3e63078d-befa-41e7-b13f-11fd3a63588a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":35,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6810","nonce":2620086826},{"type":"v1","addr":"172.21.15.89:6811","nonce":2620086826}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6812","nonce":2620086826},{"type":"v1","addr":"172.21.15.89:6813","nonce":2620086826}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6816","nonce":2620086826},{"type":"v1","addr":"172.21.15.89:6817","nonce":2620086826}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6814","nonce":2620086826},{"type":"v1","addr":"172.21.15.89:6815","nonce":2620086826}]},"public_addr":"172.21.15.89:6811/2620086826","cluster_addr":"172.21.15.89:6813/2620086826","heartbeat_back_addr":"172.21.15.89:6817/2620086826","heartbeat_front_addr":"172.21.15.89:6815/2620086826","state":["exists","up"]},{"osd":2,"uuid":"99ec6c1f-c76e-4f79-b030-b980e297e1cb","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":19,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6818","nonce":3898997901},{"type":"v1","addr":"172.21.15.89:6819","nonce":3898997901}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6820","nonce":3898997901},{"type":"v1","addr":"172.21.15.89:6821","nonce":3898997901}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6824","nonce":3898997901},{"type":"v1","addr":"172.21.15.89:6825","nonce":3898997901}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6822","nonce":3898997901},{"type":"v1","addr":"172.21.15.89:6823","nonce":3898997901}]},"public_addr":"172.21.15.89:6819/3898997901","cluster_addr":"172.21.15.89:6821/3898997901","heartbeat_back_addr":"172.21.15.89:6825/3898997901","heartbeat_front_addr":"172.21.15.89:6823/3898997901","state":["exists","up"]},{"osd":3,"uuid":"288ca940-ca66-41f1-894f-32ac0c9d9eb3","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":27,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6826","nonce":2601107822},{"type":"v1","addr":"172.21.15.89:6827","nonce":2601107822}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6828","nonce":2601107822},{"type":"v1","addr":"172.21.15.89:6829","nonce":2601107822}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6832","nonce":2601107822},{"type":"v1","addr":"172.21.15.89:6833","nonce":2601107822}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6830","nonce":2601107822},{"type":"v1","addr":"172.21.15.89:6831","nonce":2601107822}]},"public_addr":"172.21.15.89:6827/2601107822","cluster_addr":"172.21.15.89:6829/2601107822","heartbeat_back_addr":"172.21.15.89:6833/2601107822","heartbeat_front_addr":"172.21.15.89:6831/2601107822","state":["exists","up"]},{"osd":4,"uuid":"b35557d2-875d-420b-ac49-a0cfe914822d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":32,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6800","nonce":44100003},{"type":"v1","addr":"172.21.15.195:6801","nonce":44100003}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6802","nonce":44100003},{"type":"v1","addr":"172.21.15.195:6803","nonce":44100003}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6806","nonce":44100003},{"type":"v1","addr":"172.21.15.195:6807","nonce":44100003}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6804","nonce":44100003},{"type":"v1","addr":"172.21.15.195:6805","nonce":44100003}]},"public_addr":"172.21.15.195:6801/44100003","cluster_addr":"172.21.15.195:6803/44100003","heartbeat_back_addr":"172.21.15.195:6807/44100003","heartbeat_front_addr":"172.21.15.195:6805/44100003","state":["exists","up"]},{"osd":5,"uuid":"94ffbc65-1d43-460b-84eb-6be843a72bae","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":40,"up_thru":43,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6808","nonce":1913588562},{"type":"v1","addr":"172.21.15.195:6809","nonce":1913588562}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6810","nonce":1913588562},{"type":"v1","addr":"172.21.15.195:6811","nonce":1913588562}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6814","nonce":1913588562},{"type":"v1","addr":"172.21.15.195:6815","nonce":1913588562}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6812","nonce":1913588562},{"type":"v1","addr":"172.21.15.195:6813","nonce":1913588562}]},"public_addr":"172.21.15.195:6809/1913588562","cluster_addr":"172.21.15.195:6811/1913588562","heartbeat_back_addr":"172.21.15.195:6815/1913588562","heartbeat_front_addr":"172.21.15.195:6813/1913588562","state":["exists","up"]},{"osd":6,"uuid":"3ae73a35-8e39-47b1-8f1b-e52c81bbf9c8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":48,"up_thru":51,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6816","nonce":1620163911},{"type":"v1","addr":"172.21.15.195:6817","nonce":1620163911}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6818","nonce":1620163911},{"type":"v1","addr":"172.21.15.195:6819","nonce":1620163911}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6822","nonce":1620163911},{"type":"v1","addr":"172.21.15.195:6823","nonce":1620163911}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6820","nonce":1620163911},{"type":"v1","addr":"172.21.15.195:6821","nonce":1620163911}]},"public_addr":"172.21.15.195:6817/1620163911","cluster_addr":"172.21.15.195:6819/1620163911","heartbeat_back_addr":"172.21.15.195:6823/1620163911","heartbeat_front_addr":"172.21.15.195:6821/1620163911","state":["exists","up"]},{"osd":7,"uuid":"50ea76bf-c840-4b8b-9910-dad0e4aafc23","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":56,"up_thru":59,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6824","nonce":2424689581},{"type":"v1","addr":"172.21.15.195:6825","nonce":2424689581}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6826","nonce":2424689581},{"type":"v1","addr":"172.21.15.195:6827","nonce":2424689581}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6830","nonce":2424689581},{"type":"v1","addr":"172.21.15.195:6831","nonce":2424689581}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6828","nonce":2424689581},{"type":"v1","addr":"172.21.15.195:6829","nonce":2424689581}]},"public_addr":"172.21.15.195:6825/2424689581","cluster_addr":"172.21.15.195:6827/2424689581","heartbeat_back_addr":"172.21.15.195:6831/2424689581","heartbeat_front_addr":"172.21.15.195:6829/2424689581","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T04:04:18.651040+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T04:04:48.404481+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T04:05:16.848563+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T04:05:44.741921+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T04:06:06.786033+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T04:06:30.128500+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T04:06:54.330921+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T04:07:18.049029+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.89:6801/1123747776":"2024-09-22T04:03:13.037420+0000","172.21.15.89:6800/1123747776":"2024-09-22T04:03:13.037420+0000","172.21.15.89:0/3294782937":"2024-09-22T04:02:24.242528+0000","172.21.15.89:0/2709663841":"2024-09-22T04:02:24.242528+0000","172.21.15.89:6801/1983486000":"2024-09-22T04:02:24.242528+0000","172.21.15.89:6800/1983486000":"2024-09-22T04:02:24.242528+0000","172.21.15.89:0/3099511971":"2024-09-22T04:01:55.211457+0000","172.21.15.89:0/328064113":"2024-09-22T04:03:13.037420+0000","172.21.15.89:0/2787795518":"2024-09-22T04:02:24.242528+0000","172.21.15.89:6800/4163677862":"2024-09-22T04:01:55.211457+0000","172.21.15.89:0/2972904652":"2024-09-22T04:01:55.211457+0000","172.21.15.89:0/3651427575":"2024-09-22T04:03:13.037420+0000","172.21.15.89:0/2159750059":"2024-09-22T04:03:13.037420+0000","172.21.15.89:6801/4163677862":"2024-09-22T04:01:55.211457+0000","172.21.15.89:0/2242250733":"2024-09-22T04:01:55.211457+0000"},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-21T04:07:43.333 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:07:43 smithi089 ceph-mon[23733]: from='client.? 172.21.15.89:0/609480093' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-21T04:07:43.356 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 1b9b79fc-77ce-11ef-baf5-efdc52797490 -- ceph tell osd.0 flush_pg_stats 2024-09-21T04:07:43.356 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 1b9b79fc-77ce-11ef-baf5-efdc52797490 -- ceph tell osd.1 flush_pg_stats 2024-09-21T04:07:43.357 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 1b9b79fc-77ce-11ef-baf5-efdc52797490 -- ceph tell osd.2 flush_pg_stats 2024-09-21T04:07:43.357 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 1b9b79fc-77ce-11ef-baf5-efdc52797490 -- ceph tell osd.3 flush_pg_stats 2024-09-21T04:07:43.358 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 1b9b79fc-77ce-11ef-baf5-efdc52797490 -- ceph tell osd.4 flush_pg_stats 2024-09-21T04:07:43.358 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 1b9b79fc-77ce-11ef-baf5-efdc52797490 -- ceph tell osd.5 flush_pg_stats 2024-09-21T04:07:43.358 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 1b9b79fc-77ce-11ef-baf5-efdc52797490 -- ceph tell osd.6 flush_pg_stats 2024-09-21T04:07:43.359 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 1b9b79fc-77ce-11ef-baf5-efdc52797490 -- ceph tell osd.7 flush_pg_stats 2024-09-21T04:07:43.608 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:07:43 smithi195 ceph-mon[25840]: from='client.? 172.21.15.89:0/609480093' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-21T04:07:43.758 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/1b9b79fc-77ce-11ef-baf5-efdc52797490/mon.smithi089/config 2024-09-21T04:07:43.759 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/1b9b79fc-77ce-11ef-baf5-efdc52797490/mon.smithi089/config 2024-09-21T04:07:43.759 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/1b9b79fc-77ce-11ef-baf5-efdc52797490/mon.smithi089/config 2024-09-21T04:07:43.762 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/1b9b79fc-77ce-11ef-baf5-efdc52797490/mon.smithi089/config 2024-09-21T04:07:43.762 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/1b9b79fc-77ce-11ef-baf5-efdc52797490/mon.smithi089/config 2024-09-21T04:07:43.763 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/1b9b79fc-77ce-11ef-baf5-efdc52797490/mon.smithi089/config 2024-09-21T04:07:43.770 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/1b9b79fc-77ce-11ef-baf5-efdc52797490/mon.smithi089/config 2024-09-21T04:07:43.785 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/1b9b79fc-77ce-11ef-baf5-efdc52797490/mon.smithi089/config 2024-09-21T04:07:44.414 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:07:44 smithi089 ceph-mon[23733]: pgmap v183: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-21T04:07:44.608 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:07:44 smithi195 ceph-mon[25840]: pgmap v183: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-21T04:07:46.264 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:07:46 smithi089 ceph-mon[23733]: pgmap v184: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-21T04:07:46.608 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:07:46 smithi195 ceph-mon[25840]: pgmap v184: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-21T04:07:48.264 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:07:48 smithi089 ceph-mon[23733]: pgmap v185: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-21T04:07:48.608 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:07:48 smithi195 ceph-mon[25840]: pgmap v185: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-21T04:07:50.264 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:07:50 smithi089 ceph-mon[23733]: pgmap v186: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-21T04:07:50.595 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:07:50 smithi195 ceph-mon[25840]: pgmap v186: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-21T04:07:52.294 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:07:52 smithi089 ceph-mon[23733]: pgmap v187: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-21T04:07:52.608 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:07:52 smithi195 ceph-mon[25840]: pgmap v187: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-21T04:07:54.514 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:07:54 smithi089 ceph-mon[23733]: pgmap v188: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-21T04:07:54.606 INFO:teuthology.orchestra.run.smithi089.stdout:115964117021 2024-09-21T04:07:54.606 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 1b9b79fc-77ce-11ef-baf5-efdc52797490 -- ceph osd last-stat-seq osd.3 2024-09-21T04:07:54.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:07:54 smithi195 ceph-mon[25840]: pgmap v188: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-21T04:07:55.511 INFO:teuthology.orchestra.run.smithi089.stdout:60129542182 2024-09-21T04:07:55.511 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 1b9b79fc-77ce-11ef-baf5-efdc52797490 -- ceph osd last-stat-seq osd.1 2024-09-21T04:07:56.514 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:07:56 smithi089 ceph-mon[23733]: pgmap v189: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-21T04:07:56.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:07:56 smithi195 ceph-mon[25840]: pgmap v189: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-21T04:07:56.626 INFO:teuthology.orchestra.run.smithi089.stdout:38654705709 2024-09-21T04:07:56.627 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 1b9b79fc-77ce-11ef-baf5-efdc52797490 -- ceph osd last-stat-seq osd.0 2024-09-21T04:07:57.239 INFO:teuthology.orchestra.run.smithi089.stdout:171798691860 2024-09-21T04:07:57.240 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 1b9b79fc-77ce-11ef-baf5-efdc52797490 -- ceph osd last-stat-seq osd.5 2024-09-21T04:07:58.258 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:07:58 smithi089 ceph-mon[23733]: pgmap v190: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-21T04:07:58.515 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/1b9b79fc-77ce-11ef-baf5-efdc52797490/mon.smithi089/config 2024-09-21T04:07:58.608 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:07:58 smithi195 ceph-mon[25840]: pgmap v190: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-21T04:07:59.046 INFO:teuthology.orchestra.run.smithi089.stdout:206158430223 2024-09-21T04:07:59.046 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 1b9b79fc-77ce-11ef-baf5-efdc52797490 -- ceph osd last-stat-seq osd.6 2024-09-21T04:07:59.692 INFO:teuthology.orchestra.run.smithi089.stdout:240518168587 2024-09-21T04:07:59.693 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 1b9b79fc-77ce-11ef-baf5-efdc52797490 -- ceph osd last-stat-seq osd.7 2024-09-21T04:08:00.391 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:08:00 smithi089 ceph-mon[23733]: pgmap v191: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-21T04:08:00.415 INFO:teuthology.orchestra.run.smithi089.stdout:137438953497 2024-09-21T04:08:00.416 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 1b9b79fc-77ce-11ef-baf5-efdc52797490 -- ceph osd last-stat-seq osd.4 2024-09-21T04:08:00.589 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/1b9b79fc-77ce-11ef-baf5-efdc52797490/mon.smithi089/config 2024-09-21T04:08:00.589 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/1b9b79fc-77ce-11ef-baf5-efdc52797490/mon.smithi089/config 2024-09-21T04:08:00.595 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:08:00 smithi195 ceph-mon[25840]: pgmap v191: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-21T04:08:00.984 INFO:teuthology.orchestra.run.smithi089.stdout:81604378660 2024-09-21T04:08:00.984 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 1b9b79fc-77ce-11ef-baf5-efdc52797490 -- ceph osd last-stat-seq osd.2 2024-09-21T04:08:01.058 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/1b9b79fc-77ce-11ef-baf5-efdc52797490/mon.smithi089/config 2024-09-21T04:08:01.058 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/1b9b79fc-77ce-11ef-baf5-efdc52797490/mon.smithi089/config 2024-09-21T04:08:01.058 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/1b9b79fc-77ce-11ef-baf5-efdc52797490/mon.smithi089/config 2024-09-21T04:08:01.062 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/1b9b79fc-77ce-11ef-baf5-efdc52797490/mon.smithi089/config 2024-09-21T04:08:02.514 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:08:02 smithi089 ceph-mon[23733]: pgmap v192: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-21T04:08:02.608 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:08:02 smithi195 ceph-mon[25840]: pgmap v192: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-21T04:08:04.440 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:08:04 smithi089 ceph-mon[23733]: pgmap v193: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-21T04:08:04.608 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:08:04 smithi195 ceph-mon[25840]: pgmap v193: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-21T04:08:04.864 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/1b9b79fc-77ce-11ef-baf5-efdc52797490/mon.smithi089/config 2024-09-21T04:08:06.151 INFO:teuthology.orchestra.run.smithi089.stdout:115964117023 2024-09-21T04:08:06.344 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:08:06 smithi089 ceph-mon[23733]: pgmap v194: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-21T04:08:06.344 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:08:06 smithi089 ceph-mon[23733]: from='client.? 172.21.15.89:0/1387448181' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-09-21T04:08:06.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:08:06 smithi195 ceph-mon[25840]: pgmap v194: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-21T04:08:06.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:08:06 smithi195 ceph-mon[25840]: from='client.? 172.21.15.89:0/1387448181' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-09-21T04:08:07.649 INFO:tasks.cephadm.ceph_manager.ceph:need seq 115964117021 got 115964117023 for osd.3 2024-09-21T04:08:07.649 DEBUG:teuthology.parallel:result is None 2024-09-21T04:08:08.419 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:08:08 smithi089 ceph-mon[23733]: pgmap v195: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-21T04:08:08.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:08:08 smithi195 ceph-mon[25840]: pgmap v195: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-21T04:08:08.622 INFO:teuthology.orchestra.run.smithi089.stdout:206158430226 2024-09-21T04:08:09.262 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:08:09 smithi089 ceph-mon[23733]: from='client.? 172.21.15.89:0/3471582432' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-09-21T04:08:09.608 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:08:09 smithi195 ceph-mon[25840]: from='client.? 172.21.15.89:0/3471582432' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-09-21T04:08:09.995 INFO:teuthology.orchestra.run.smithi089.stdout:240518168589 2024-09-21T04:08:10.217 INFO:teuthology.orchestra.run.smithi089.stdout:38654705712 2024-09-21T04:08:10.264 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:08:10 smithi089 ceph-mon[23733]: pgmap v196: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-21T04:08:10.264 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:08:10 smithi089 ceph-mon[23733]: from='client.? 172.21.15.89:0/201343086' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-09-21T04:08:10.594 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:08:10 smithi195 ceph-mon[25840]: pgmap v196: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-21T04:08:10.595 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:08:10 smithi195 ceph-mon[25840]: from='client.? 172.21.15.89:0/201343086' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-09-21T04:08:10.611 INFO:tasks.cephadm.ceph_manager.ceph:need seq 240518168587 got 240518168589 for osd.7 2024-09-21T04:08:10.611 DEBUG:teuthology.parallel:result is None 2024-09-21T04:08:11.348 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:08:11 smithi089 ceph-mon[23733]: from='client.? 172.21.15.89:0/151695417' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-09-21T04:08:11.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:08:11 smithi195 ceph-mon[25840]: from='client.? 172.21.15.89:0/151695417' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-09-21T04:08:12.307 INFO:teuthology.orchestra.run.smithi089.stdout:81604378662 2024-09-21T04:08:12.511 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:08:12 smithi089 ceph-mon[23733]: pgmap v197: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-21T04:08:12.608 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:08:12 smithi195 ceph-mon[25840]: pgmap v197: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-21T04:08:13.075 INFO:tasks.cephadm.ceph_manager.ceph:need seq 81604378660 got 81604378662 for osd.2 2024-09-21T04:08:13.075 DEBUG:teuthology.parallel:result is None 2024-09-21T04:08:13.334 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:08:13 smithi089 ceph-mon[23733]: from='client.? 172.21.15.89:0/2914734961' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-09-21T04:08:13.334 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:08:13 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi089.srlyxf/mirror_snapshot_schedule"}]: dispatch 2024-09-21T04:08:13.335 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:08:13 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi089.srlyxf/trash_purge_schedule"}]: dispatch 2024-09-21T04:08:13.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:08:13 smithi195 ceph-mon[25840]: from='client.? 172.21.15.89:0/2914734961' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-09-21T04:08:13.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:08:13 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi089.srlyxf/mirror_snapshot_schedule"}]: dispatch 2024-09-21T04:08:13.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:08:13 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi089.srlyxf/trash_purge_schedule"}]: dispatch 2024-09-21T04:08:13.695 INFO:tasks.cephadm.ceph_manager.ceph:need seq 206158430223 got 206158430226 for osd.6 2024-09-21T04:08:13.695 DEBUG:teuthology.parallel:result is None 2024-09-21T04:08:14.072 INFO:teuthology.orchestra.run.smithi089.stdout:137438953500 2024-09-21T04:08:14.264 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:08:14 smithi089 ceph-mon[23733]: pgmap v198: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-21T04:08:14.264 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:08:14 smithi089 ceph-mon[23733]: from='client.? 172.21.15.89:0/3470902250' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-09-21T04:08:14.345 INFO:tasks.cephadm.ceph_manager.ceph:need seq 38654705709 got 38654705712 for osd.0 2024-09-21T04:08:14.345 DEBUG:teuthology.parallel:result is None 2024-09-21T04:08:14.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:08:14 smithi195 ceph-mon[25840]: pgmap v198: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-21T04:08:14.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:08:14 smithi195 ceph-mon[25840]: from='client.? 172.21.15.89:0/3470902250' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-09-21T04:08:15.812 INFO:teuthology.orchestra.run.smithi089.stdout:171798691864 2024-09-21T04:08:15.949 INFO:tasks.cephadm.ceph_manager.ceph:need seq 137438953497 got 137438953500 for osd.4 2024-09-21T04:08:15.949 DEBUG:teuthology.parallel:result is None 2024-09-21T04:08:16.500 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:08:16 smithi089 ceph-mon[23733]: pgmap v199: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-21T04:08:16.501 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:08:16 smithi089 ceph-mon[23733]: from='client.? 172.21.15.89:0/1131017568' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-09-21T04:08:16.522 INFO:tasks.cephadm.ceph_manager.ceph:need seq 171798691860 got 171798691864 for osd.5 2024-09-21T04:08:16.522 DEBUG:teuthology.parallel:result is None 2024-09-21T04:08:16.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:08:16 smithi195 ceph-mon[25840]: pgmap v199: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-21T04:08:16.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:08:16 smithi195 ceph-mon[25840]: from='client.? 172.21.15.89:0/1131017568' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-09-21T04:08:16.951 INFO:teuthology.orchestra.run.smithi089.stdout:60129542187 2024-09-21T04:08:17.514 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:08:17 smithi089 ceph-mon[23733]: from='client.? 172.21.15.89:0/2012807992' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-09-21T04:08:17.560 INFO:tasks.cephadm.ceph_manager.ceph:need seq 60129542182 got 60129542187 for osd.1 2024-09-21T04:08:17.561 DEBUG:teuthology.parallel:result is None 2024-09-21T04:08:17.561 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2024-09-21T04:08:17.561 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 1b9b79fc-77ce-11ef-baf5-efdc52797490 -- ceph pg dump --format=json 2024-09-21T04:08:17.608 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:08:17 smithi195 ceph-mon[25840]: from='client.? 172.21.15.89:0/2012807992' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-09-21T04:08:17.766 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/1b9b79fc-77ce-11ef-baf5-efdc52797490/mon.smithi089/config 2024-09-21T04:08:18.514 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:08:18 smithi089 ceph-mon[23733]: pgmap v200: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-21T04:08:18.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:08:18 smithi195 ceph-mon[25840]: pgmap v200: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-21T04:08:19.524 INFO:teuthology.orchestra.run.smithi089.stdout: 2024-09-21T04:08:19.526 INFO:teuthology.orchestra.run.smithi089.stderr:dumped all 2024-09-21T04:08:20.067 INFO:teuthology.orchestra.run.smithi089.stdout:{"pg_ready":true,"pg_map":{"version":201,"stamp":"2024-09-21T04:08:19.119003+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":117,"ondisk_log_size":117,"up":3,"acting":3,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":3,"num_osds":8,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":749961216,"kb_used":48928,"kb_used_data":6368,"kb_used_omap":0,"kb_used_meta":42496,"kb_avail":749912288,"statfs":{"total":767960285184,"available":767910182912,"internally_reserved":0,"allocated":6520832,"data_stored":3571206,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":43515904},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002245"},"pg_stats":[{"pgid":"1.0","version":"23'117","reported_seq":226,"reported_epoch":60,"state":"active+clean","last_fresh":"2024-09-21T04:07:24.122732+0000","last_change":"2024-09-21T04:07:24.122732+0000","last_active":"2024-09-21T04:07:24.122732+0000","last_peered":"2024-09-21T04:07:24.122732+0000","last_clean":"2024-09-21T04:07:24.122732+0000","last_became_active":"2024-09-21T04:07:24.122284+0000","last_became_peered":"2024-09-21T04:07:24.122284+0000","last_unstale":"2024-09-21T04:07:24.122732+0000","last_undegraded":"2024-09-21T04:07:24.122732+0000","last_fullsized":"2024-09-21T04:07:24.122732+0000","mapping_epoch":59,"log_start":"0'0","ondisk_log_start":"0'0","created":21,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T04:05:19.905898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T04:05:19.905898+0000","last_clean_scrub_stamp":"2024-09-21T04:05:19.905898+0000","objects_scrubbed":0,"log_size":117,"ondisk_log_size":117,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T04:17:08.405355+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]}],"pool_stats":[{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":3727360,"data_stored":3702496,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":117,"ondisk_log_size":117,"up":3,"acting":3,"num_store_stats":7}],"osd_stats":[{"osd":7,"up_from":56,"seq":240518168591,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6056,"kb_used_data":1120,"kb_used_omap":0,"kb_used_meta":4928,"kb_avail":93739096,"statfs":{"total":95995035648,"available":95988834304,"internally_reserved":0,"allocated":1146880,"data_stored":776872,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5046272},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.51900000000000002}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.377}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.505}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44900000000000001}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.48499999999999999}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.55200000000000005}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.60299999999999998}]}]},{"osd":6,"up_from":48,"seq":206158430228,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6180,"kb_used_data":1116,"kb_used_omap":0,"kb_used_meta":5056,"kb_avail":93738972,"statfs":{"total":95995035648,"available":95988707328,"internally_reserved":0,"allocated":1142784,"data_stored":776607,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5177344},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 21 04:07:58 2024","interfaces":[{"interface":"back","average":{"1min":1.613,"5min":1.613,"15min":1.613},"min":{"1min":0.41499999999999998,"5min":0.41499999999999998,"15min":0.41499999999999998},"max":{"1min":14.903,"5min":14.903,"15min":14.903},"last":0.64000000000000001},{"interface":"front","average":{"1min":1.635,"5min":1.635,"15min":1.635},"min":{"1min":0.40699999999999997,"5min":0.40699999999999997,"15min":0.40699999999999997},"max":{"1min":14.923999999999999,"5min":14.923999999999999,"15min":14.923999999999999},"last":0.56599999999999995}]},{"osd":1,"last update":"Sat Sep 21 04:07:58 2024","interfaces":[{"interface":"back","average":{"1min":1.294,"5min":1.294,"15min":1.294},"min":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":14.885,"5min":14.885,"15min":14.885},"last":0.622},{"interface":"front","average":{"1min":0.54200000000000004,"5min":0.54200000000000004,"15min":0.54200000000000004},"min":{"1min":0.38300000000000001,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":0.83599999999999997,"5min":0.83599999999999997,"15min":0.83599999999999997},"last":0.53800000000000003}]},{"osd":2,"last update":"Sat Sep 21 04:07:58 2024","interfaces":[{"interface":"back","average":{"1min":0.53400000000000003,"5min":0.53400000000000003,"15min":0.53400000000000003},"min":{"1min":0.40000000000000002,"5min":0.40000000000000002,"15min":0.40000000000000002},"max":{"1min":0.78300000000000003,"5min":0.78300000000000003,"15min":0.78300000000000003},"last":0.60799999999999998},{"interface":"front","average":{"1min":0.56799999999999995,"5min":0.56799999999999995,"15min":0.56799999999999995},"min":{"1min":0.38900000000000001,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":0.76900000000000002,"5min":0.76900000000000002,"15min":0.76900000000000002},"last":0.58699999999999997}]},{"osd":3,"last update":"Sat Sep 21 04:07:58 2024","interfaces":[{"interface":"back","average":{"1min":0.95799999999999996,"5min":0.95799999999999996,"15min":0.95799999999999996},"min":{"1min":0.437,"5min":0.437,"15min":0.437},"max":{"1min":8.1159999999999997,"5min":8.1159999999999997,"15min":8.1159999999999997},"last":0.59799999999999998},{"interface":"front","average":{"1min":0.55900000000000005,"5min":0.55900000000000005,"15min":0.55900000000000005},"min":{"1min":0.374,"5min":0.374,"15min":0.374},"max":{"1min":0.746,"5min":0.746,"15min":0.746},"last":0.57599999999999996}]},{"osd":4,"last update":"Sat Sep 21 04:07:58 2024","interfaces":[{"interface":"back","average":{"1min":0.432,"5min":0.432,"15min":0.432},"min":{"1min":0.27500000000000002,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":0.70999999999999996,"5min":0.70999999999999996,"15min":0.70999999999999996},"last":0.42499999999999999},{"interface":"front","average":{"1min":0.47299999999999998,"5min":0.47299999999999998,"15min":0.47299999999999998},"min":{"1min":0.26600000000000001,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":0.80400000000000005,"5min":0.80400000000000005,"15min":0.80400000000000005},"last":0.47899999999999998}]},{"osd":5,"last update":"Sat Sep 21 04:07:58 2024","interfaces":[{"interface":"back","average":{"1min":0.46800000000000003,"5min":0.46800000000000003,"15min":0.46800000000000003},"min":{"1min":0.31,"5min":0.31,"15min":0.31},"max":{"1min":0.72599999999999998,"5min":0.72599999999999998,"15min":0.72599999999999998},"last":0.51100000000000001},{"interface":"front","average":{"1min":0.48899999999999999,"5min":0.48899999999999999,"15min":0.48899999999999999},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":0.79200000000000004,"5min":0.79200000000000004,"15min":0.79200000000000004},"last":0.52700000000000002}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.55200000000000005}]}]},{"osd":1,"up_from":14,"seq":60129542188,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":6372,"kb_used_data":604,"kb_used_omap":0,"kb_used_meta":5760,"kb_avail":93738780,"statfs":{"total":95995035648,"available":95988510720,"internally_reserved":0,"allocated":618496,"data_stored":248267,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5898240},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 21 04:07:55 2024","interfaces":[{"interface":"back","average":{"1min":0.34699999999999998,"5min":0.33700000000000002,"15min":0.32500000000000001},"min":{"1min":0.22900000000000001,"5min":0.20399999999999999,"15min":0.20399999999999999},"max":{"1min":0.503,"5min":0.60699999999999998,"15min":0.60699999999999998},"last":0.28499999999999998},{"interface":"front","average":{"1min":0.35299999999999998,"5min":0.34300000000000003,"15min":0.32300000000000001},"min":{"1min":0.24099999999999999,"5min":0.193,"15min":0.193},"max":{"1min":0.60199999999999998,"5min":0.64000000000000001,"15min":0.64000000000000001},"last":0.33700000000000002}]},{"osd":2,"last update":"Sat Sep 21 04:07:24 2024","interfaces":[{"interface":"back","average":{"1min":0.40999999999999998,"5min":0.35099999999999998,"15min":0.34100000000000003},"min":{"1min":0.27300000000000002,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":0.53900000000000003,"5min":0.53900000000000003,"15min":0.53900000000000003},"last":0.35099999999999998},{"interface":"front","average":{"1min":0.41099999999999998,"5min":0.36199999999999999,"15min":0.35399999999999998},"min":{"1min":0.30499999999999999,"5min":0.25,"15min":0.25},"max":{"1min":0.52700000000000002,"5min":0.58899999999999997,"15min":0.58899999999999997},"last":0.26800000000000002}]},{"osd":3,"last update":"Sat Sep 21 04:07:55 2024","interfaces":[{"interface":"back","average":{"1min":0.42399999999999999,"5min":0.45100000000000001,"15min":0.45500000000000002},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":0.64900000000000002,"5min":0.65900000000000003,"15min":0.65900000000000003},"last":0.30499999999999999},{"interface":"front","average":{"1min":0.40600000000000003,"5min":0.42499999999999999,"15min":0.42799999999999999},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":0.55100000000000005,"5min":0.63300000000000001,"15min":0.63300000000000001},"last":0.314}]},{"osd":4,"last update":"Sat Sep 21 04:08:13 2024","interfaces":[{"interface":"back","average":{"1min":0.56699999999999995,"5min":0.58799999999999997,"15min":0.59199999999999997},"min":{"1min":0.42799999999999999,"5min":0.41999999999999998,"15min":0.41999999999999998},"max":{"1min":0.70999999999999996,"5min":0.78500000000000003,"15min":0.78500000000000003},"last":0.60599999999999998},{"interface":"front","average":{"1min":0.58599999999999997,"5min":0.56499999999999995,"15min":0.56100000000000005},"min":{"1min":0.42799999999999999,"5min":0.42799999999999999,"15min":0.42799999999999999},"max":{"1min":0.79200000000000004,"5min":0.79200000000000004,"15min":0.79200000000000004},"last":0.54100000000000004}]},{"osd":5,"last update":"Sat Sep 21 04:07:33 2024","interfaces":[{"interface":"back","average":{"1min":0.56399999999999995,"5min":0.56399999999999995,"15min":0.56399999999999995},"min":{"1min":0.433,"5min":0.433,"15min":0.433},"max":{"1min":0.80900000000000005,"5min":0.80900000000000005,"15min":0.80900000000000005},"last":0.59199999999999997},{"interface":"front","average":{"1min":0.58399999999999996,"5min":0.58399999999999996,"15min":0.58399999999999996},"min":{"1min":0.45000000000000001,"5min":0.45000000000000001,"15min":0.45000000000000001},"max":{"1min":0.71499999999999997,"5min":0.71499999999999997,"15min":0.71499999999999997},"last":0.52400000000000002}]},{"osd":6,"last update":"Sat Sep 21 04:07:56 2024","interfaces":[{"interface":"back","average":{"1min":0.58999999999999997,"5min":0.58999999999999997,"15min":0.58999999999999997},"min":{"1min":0.46300000000000002,"5min":0.46300000000000002,"15min":0.46300000000000002},"max":{"1min":0.74099999999999999,"5min":0.74099999999999999,"15min":0.74099999999999999},"last":0.57099999999999995},{"interface":"front","average":{"1min":0.59999999999999998,"5min":0.59999999999999998,"15min":0.59999999999999998},"min":{"1min":0.495,"5min":0.495,"15min":0.495},"max":{"1min":0.85099999999999998,"5min":0.85099999999999998,"15min":0.85099999999999998},"last":0.56299999999999994}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.54600000000000004}]}]},{"osd":0,"up_from":9,"seq":38654705714,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6956,"kb_used_data":1124,"kb_used_omap":0,"kb_used_meta":5824,"kb_avail":93738196,"statfs":{"total":95995035648,"available":95987912704,"internally_reserved":0,"allocated":1150976,"data_stored":777195,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5963776},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 21 04:08:02 2024","interfaces":[{"interface":"back","average":{"1min":0.34200000000000003,"5min":0.34100000000000003,"15min":0.33400000000000002},"min":{"1min":0.189,"5min":0.189,"15min":0.189},"max":{"1min":0.47199999999999998,"5min":0.48399999999999999,"15min":0.48399999999999999},"last":0.28299999999999997},{"interface":"front","average":{"1min":0.33900000000000002,"5min":0.33300000000000002,"15min":0.32400000000000001},"min":{"1min":0.22,"5min":0.22,"15min":0.22},"max":{"1min":0.52500000000000002,"5min":0.52500000000000002,"15min":0.52500000000000002},"last":0.35299999999999998}]},{"osd":2,"last update":"Sat Sep 21 04:07:24 2024","interfaces":[{"interface":"back","average":{"1min":0.40999999999999998,"5min":0.35999999999999999,"15min":0.35199999999999998},"min":{"1min":0.33500000000000002,"5min":0.217,"15min":0.217},"max":{"1min":0.52700000000000002,"5min":0.55100000000000005,"15min":0.55100000000000005},"last":0.32600000000000001},{"interface":"front","average":{"1min":0.38800000000000001,"5min":0.36399999999999999,"15min":0.35999999999999999},"min":{"1min":0.248,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":0.50700000000000001,"5min":0.62,"15min":0.62},"last":0.33400000000000002}]},{"osd":3,"last update":"Sat Sep 21 04:07:47 2024","interfaces":[{"interface":"back","average":{"1min":0.372,"5min":0.39400000000000002,"15min":0.39800000000000002},"min":{"1min":0.23799999999999999,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":0.50700000000000001,"5min":0.53900000000000003,"15min":0.53900000000000003},"last":0.29399999999999998},{"interface":"front","average":{"1min":0.40799999999999997,"5min":0.43099999999999999,"15min":0.435},"min":{"1min":0.31,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":0.55700000000000005,"5min":0.57499999999999996,"15min":0.57499999999999996},"last":0.38100000000000001}]},{"osd":4,"last update":"Sat Sep 21 04:08:11 2024","interfaces":[{"interface":"back","average":{"1min":0.59599999999999997,"5min":0.56000000000000005,"15min":0.55400000000000005},"min":{"1min":0.46000000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":0.79400000000000004,"5min":0.85699999999999998,"15min":0.85699999999999998},"last":0.47599999999999998},{"interface":"front","average":{"1min":0.59999999999999998,"5min":1.21,"15min":1.3120000000000001},"min":{"1min":0.371,"5min":0.371,"15min":0.371},"max":{"1min":0.81799999999999995,"5min":14.523,"15min":14.523},"last":0.39800000000000002}]},{"osd":5,"last update":"Sat Sep 21 04:07:35 2024","interfaces":[{"interface":"back","average":{"1min":0.58799999999999997,"5min":0.58799999999999997,"15min":0.58799999999999997},"min":{"1min":0.42999999999999999,"5min":0.42999999999999999,"15min":0.42999999999999999},"max":{"1min":0.73499999999999999,"5min":0.73499999999999999,"15min":0.73499999999999999},"last":0.442},{"interface":"front","average":{"1min":0.59199999999999997,"5min":0.59199999999999997,"15min":0.59199999999999997},"min":{"1min":0.39700000000000002,"5min":0.39700000000000002,"15min":0.39700000000000002},"max":{"1min":0.82399999999999995,"5min":0.82399999999999995,"15min":0.82399999999999995},"last":0.59499999999999997}]},{"osd":6,"last update":"Sat Sep 21 04:07:56 2024","interfaces":[{"interface":"back","average":{"1min":0.58199999999999996,"5min":0.58199999999999996,"15min":0.58199999999999996},"min":{"1min":0.44400000000000001,"5min":0.44400000000000001,"15min":0.44400000000000001},"max":{"1min":0.79800000000000004,"5min":0.79800000000000004,"15min":0.79800000000000004},"last":0.498},{"interface":"front","average":{"1min":0.59099999999999997,"5min":0.59099999999999997,"15min":0.59099999999999997},"min":{"1min":0.439,"5min":0.439,"15min":0.439},"max":{"1min":0.78800000000000003,"5min":0.78800000000000003,"15min":0.78800000000000003},"last":0.42499999999999999}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.46899999999999997}]}]},{"osd":2,"up_from":19,"seq":81604378664,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":6240,"kb_used_data":600,"kb_used_omap":0,"kb_used_meta":5632,"kb_avail":93738912,"statfs":{"total":95995035648,"available":95988645888,"internally_reserved":0,"allocated":614400,"data_stored":248002,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5767168},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 21 04:07:23 2024","interfaces":[{"interface":"back","average":{"1min":0.40999999999999998,"5min":0.38200000000000001,"15min":0.377},"min":{"1min":0.29199999999999998,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":0.56299999999999994,"5min":0.78000000000000003,"15min":0.78000000000000003},"last":0.49299999999999999},{"interface":"front","average":{"1min":0.41599999999999998,"5min":0.38400000000000001,"15min":0.379},"min":{"1min":0.28000000000000003,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":0.55900000000000005,"5min":0.70199999999999996,"15min":0.70199999999999996},"last":0.505}]},{"osd":1,"last update":"Sat Sep 21 04:07:23 2024","interfaces":[{"interface":"back","average":{"1min":0.443,"5min":0.39300000000000002,"15min":0.38500000000000001},"min":{"1min":0.29999999999999999,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":0.56299999999999994,"5min":0.60199999999999998,"15min":0.60199999999999998},"last":0.52800000000000002},{"interface":"front","average":{"1min":0.42999999999999999,"5min":0.375,"15min":0.36599999999999999},"min":{"1min":0.32700000000000001,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":0.53500000000000003,"5min":0.53500000000000003,"15min":0.53500000000000003},"last":0.52300000000000002}]},{"osd":3,"last update":"Sat Sep 21 04:07:52 2024","interfaces":[{"interface":"back","average":{"1min":0.432,"5min":0.441,"15min":0.443},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":0.68999999999999995,"5min":0.68999999999999995,"15min":0.68999999999999995},"last":0.56399999999999995},{"interface":"front","average":{"1min":0.41899999999999998,"5min":0.45200000000000001,"15min":0.45800000000000002},"min":{"1min":0.34499999999999997,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":0.63,"5min":0.63,"15min":0.63},"last":0.47099999999999997}]},{"osd":4,"last update":"Sat Sep 21 04:08:11 2024","interfaces":[{"interface":"back","average":{"1min":0.65200000000000002,"5min":0.79200000000000004,"15min":0.81599999999999995},"min":{"1min":0.47899999999999998,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":1.2350000000000001,"5min":6.2089999999999996,"15min":6.2089999999999996},"last":0.75800000000000001},{"interface":"front","average":{"1min":0.65900000000000003,"5min":0.60899999999999999,"15min":0.60099999999999998},"min":{"1min":0.44500000000000001,"5min":0.40899999999999997,"15min":0.40899999999999997},"max":{"1min":1.246,"5min":1.246,"15min":1.246},"last":0.71799999999999997}]},{"osd":5,"last update":"Sat Sep 21 04:07:35 2024","interfaces":[{"interface":"back","average":{"1min":0.57299999999999995,"5min":0.57299999999999995,"15min":0.57299999999999995},"min":{"1min":0.41799999999999998,"5min":0.41799999999999998,"15min":0.41799999999999998},"max":{"1min":0.85699999999999998,"5min":0.85699999999999998,"15min":0.85699999999999998},"last":0.627},{"interface":"front","average":{"1min":0.59499999999999997,"5min":0.59499999999999997,"15min":0.59499999999999997},"min":{"1min":0.42199999999999999,"5min":0.42199999999999999,"15min":0.42199999999999999},"max":{"1min":0.745,"5min":0.745,"15min":0.745},"last":0.78200000000000003}]},{"osd":6,"last update":"Sat Sep 21 04:07:58 2024","interfaces":[{"interface":"back","average":{"1min":0.64600000000000002,"5min":0.64600000000000002,"15min":0.64600000000000002},"min":{"1min":0.439,"5min":0.439,"15min":0.439},"max":{"1min":1.3640000000000001,"5min":1.3640000000000001,"15min":1.3640000000000001},"last":0.82299999999999995},{"interface":"front","average":{"1min":0.628,"5min":0.628,"15min":0.628},"min":{"1min":0.434,"5min":0.434,"15min":0.434},"max":{"1min":1.1970000000000001,"5min":1.1970000000000001,"15min":1.1970000000000001},"last":0.68500000000000005}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.74099999999999999}]}]},{"osd":3,"up_from":27,"seq":115964117026,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5732,"kb_used_data":604,"kb_used_omap":0,"kb_used_meta":5120,"kb_avail":93739420,"statfs":{"total":95995035648,"available":95989166080,"internally_reserved":0,"allocated":618496,"data_stored":248267,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5242880},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 21 04:07:52 2024","interfaces":[{"interface":"back","average":{"1min":0.39700000000000002,"5min":0.40899999999999997,"15min":0.41099999999999998},"min":{"1min":0.245,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":0.56699999999999995,"5min":0.61099999999999999,"15min":0.61099999999999999},"last":0.39700000000000002},{"interface":"front","average":{"1min":0.41699999999999998,"5min":0.39000000000000001,"15min":0.38600000000000001},"min":{"1min":0.29199999999999998,"5min":0.252,"15min":0.252},"max":{"1min":0.60199999999999998,"5min":0.60199999999999998,"15min":0.60199999999999998},"last":0.32300000000000001}]},{"osd":1,"last update":"Sat Sep 21 04:07:52 2024","interfaces":[{"interface":"back","average":{"1min":0.42699999999999999,"5min":0.41899999999999998,"15min":0.41799999999999998},"min":{"1min":0.23799999999999999,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":0.67600000000000005,"5min":0.67600000000000005,"15min":0.67600000000000005},"last":0.40699999999999997},{"interface":"front","average":{"1min":0.43099999999999999,"5min":0.40300000000000002,"15min":0.39800000000000002},"min":{"1min":0.27400000000000002,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":0.59699999999999998,"5min":0.59699999999999998,"15min":0.59699999999999998},"last":0.35199999999999998}]},{"osd":2,"last update":"Sat Sep 21 04:07:52 2024","interfaces":[{"interface":"back","average":{"1min":0.45500000000000002,"5min":0.44500000000000001,"15min":0.443},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":0.63800000000000001,"5min":0.65600000000000003,"15min":0.65600000000000003},"last":0.42499999999999999},{"interface":"front","average":{"1min":0.44600000000000001,"5min":0.41999999999999998,"15min":0.41599999999999998},"min":{"1min":0.28499999999999998,"5min":0.252,"15min":0.252},"max":{"1min":0.626,"5min":0.626,"15min":0.626},"last":0.38400000000000001}]},{"osd":4,"last update":"Sat Sep 21 04:08:13 2024","interfaces":[{"interface":"back","average":{"1min":0.56100000000000005,"5min":0.58399999999999996,"15min":0.58799999999999997},"min":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":0.74399999999999999,"5min":0.91900000000000004,"15min":0.91900000000000004},"last":0.628},{"interface":"front","average":{"1min":0.59499999999999997,"5min":0.60499999999999998,"15min":0.60699999999999998},"min":{"1min":0.40699999999999997,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":0.81999999999999995,"5min":0.89000000000000001,"15min":0.89000000000000001},"last":0.58999999999999997}]},{"osd":5,"last update":"Sat Sep 21 04:07:34 2024","interfaces":[{"interface":"back","average":{"1min":0.60799999999999998,"5min":0.60799999999999998,"15min":0.60799999999999998},"min":{"1min":0.47899999999999998,"5min":0.47899999999999998,"15min":0.47899999999999998},"max":{"1min":0.71999999999999997,"5min":0.71999999999999997,"15min":0.71999999999999997},"last":0.61799999999999999},{"interface":"front","average":{"1min":0.60199999999999998,"5min":0.60199999999999998,"15min":0.60199999999999998},"min":{"1min":0.42199999999999999,"5min":0.42199999999999999,"15min":0.42199999999999999},"max":{"1min":0.90700000000000003,"5min":0.90700000000000003,"15min":0.90700000000000003},"last":0.57599999999999996}]},{"osd":6,"last update":"Sat Sep 21 04:07:58 2024","interfaces":[{"interface":"back","average":{"1min":0.623,"5min":0.623,"15min":0.623},"min":{"1min":0.39700000000000002,"5min":0.39700000000000002,"15min":0.39700000000000002},"max":{"1min":0.78000000000000003,"5min":0.78000000000000003,"15min":0.78000000000000003},"last":0.56699999999999995},{"interface":"front","average":{"1min":0.66700000000000004,"5min":0.66700000000000004,"15min":0.66700000000000004},"min":{"1min":0.42799999999999999,"5min":0.42799999999999999,"15min":0.42799999999999999},"max":{"1min":0.876,"5min":0.876,"15min":0.876},"last":0.55900000000000005}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.54600000000000004}]}]},{"osd":4,"up_from":32,"seq":137438953501,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5728,"kb_used_data":600,"kb_used_omap":0,"kb_used_meta":5120,"kb_avail":93739424,"statfs":{"total":95995035648,"available":95989170176,"internally_reserved":0,"allocated":614400,"data_stored":248002,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5242880},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 21 04:07:14 2024","interfaces":[{"interface":"back","average":{"1min":0.50700000000000001,"5min":0.50700000000000001,"15min":0.50700000000000001},"min":{"1min":0.34799999999999998,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":0.80700000000000005,"5min":0.80700000000000005,"15min":0.80700000000000005},"last":0.59899999999999998},{"interface":"front","average":{"1min":0.505,"5min":0.505,"15min":0.505},"min":{"1min":0.38100000000000001,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":0.79200000000000004,"5min":0.79200000000000004,"15min":0.79200000000000004},"last":0.63600000000000001}]},{"osd":1,"last update":"Sat Sep 21 04:07:14 2024","interfaces":[{"interface":"back","average":{"1min":0.52700000000000002,"5min":0.52700000000000002,"15min":0.52700000000000002},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":0.753,"5min":0.753,"15min":0.753},"last":0.61299999999999999},{"interface":"front","average":{"1min":0.503,"5min":0.503,"15min":0.503},"min":{"1min":0.31,"5min":0.31,"15min":0.31},"max":{"1min":0.78300000000000003,"5min":0.78300000000000003,"15min":0.78300000000000003},"last":0.66100000000000003}]},{"osd":2,"last update":"Sat Sep 21 04:07:14 2024","interfaces":[{"interface":"back","average":{"1min":0.52400000000000002,"5min":0.52400000000000002,"15min":0.52400000000000002},"min":{"1min":0.372,"5min":0.372,"15min":0.372},"max":{"1min":0.75900000000000001,"5min":0.75900000000000001,"15min":0.75900000000000001},"last":0.58399999999999996},{"interface":"front","average":{"1min":0.53000000000000003,"5min":0.53000000000000003,"15min":0.53000000000000003},"min":{"1min":0.376,"5min":0.376,"15min":0.376},"max":{"1min":0.72499999999999998,"5min":0.72499999999999998,"15min":0.72499999999999998},"last":0.67400000000000004}]},{"osd":3,"last update":"Sat Sep 21 04:07:14 2024","interfaces":[{"interface":"back","average":{"1min":0.53300000000000003,"5min":0.53300000000000003,"15min":0.53300000000000003},"min":{"1min":0.35299999999999998,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":0.73899999999999999,"5min":0.73899999999999999,"15min":0.73899999999999999},"last":0.68600000000000005},{"interface":"front","average":{"1min":0.50600000000000001,"5min":0.50600000000000001,"15min":0.50600000000000001},"min":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":0.72599999999999998,"5min":0.72599999999999998,"15min":0.72599999999999998},"last":0.71499999999999997}]},{"osd":5,"last update":"Sat Sep 21 04:07:35 2024","interfaces":[{"interface":"back","average":{"1min":0.42899999999999999,"5min":0.42899999999999999,"15min":0.42899999999999999},"min":{"1min":0.23000000000000001,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":0.70499999999999996,"5min":0.70499999999999996,"15min":0.70499999999999996},"last":0.55800000000000005},{"interface":"front","average":{"1min":0.45400000000000001,"5min":0.45400000000000001,"15min":0.45400000000000001},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":0.81699999999999995,"5min":0.81699999999999995,"15min":0.81699999999999995},"last":0.72799999999999998}]},{"osd":6,"last update":"Sat Sep 21 04:07:59 2024","interfaces":[{"interface":"back","average":{"1min":0.49199999999999999,"5min":0.49199999999999999,"15min":0.49199999999999999},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":0.75,"5min":0.75,"15min":0.75},"last":0.77700000000000002},{"interface":"front","average":{"1min":0.504,"5min":0.504,"15min":0.504},"min":{"1min":0.29499999999999998,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":0.751,"5min":0.751,"15min":0.751},"last":0.70299999999999996}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.76100000000000001}]}]},{"osd":5,"up_from":40,"seq":171798691865,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5664,"kb_used_data":600,"kb_used_omap":0,"kb_used_meta":5056,"kb_avail":93739488,"statfs":{"total":95995035648,"available":95989235712,"internally_reserved":0,"allocated":614400,"data_stored":247994,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5177344},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 21 04:07:34 2024","interfaces":[{"interface":"back","average":{"1min":0.53000000000000003,"5min":0.53000000000000003,"15min":0.53000000000000003},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":0.73999999999999999,"5min":0.73999999999999999,"15min":0.73999999999999999},"last":0.56499999999999995},{"interface":"front","average":{"1min":0.55500000000000005,"5min":0.55500000000000005,"15min":0.55500000000000005},"min":{"1min":0.39200000000000002,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":0.90800000000000003,"5min":0.90800000000000003,"15min":0.90800000000000003},"last":0.52500000000000002}]},{"osd":1,"last update":"Sat Sep 21 04:07:34 2024","interfaces":[{"interface":"back","average":{"1min":0.52400000000000002,"5min":0.52400000000000002,"15min":0.52400000000000002},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":0.85099999999999998,"5min":0.85099999999999998,"15min":0.85099999999999998},"last":0.623},{"interface":"front","average":{"1min":0.55700000000000005,"5min":0.55700000000000005,"15min":0.55700000000000005},"min":{"1min":0.42799999999999999,"5min":0.42799999999999999,"15min":0.42799999999999999},"max":{"1min":0.81299999999999994,"5min":0.81299999999999994,"15min":0.81299999999999994},"last":0.49399999999999999}]},{"osd":2,"last update":"Sat Sep 21 04:07:34 2024","interfaces":[{"interface":"back","average":{"1min":0.54400000000000004,"5min":0.54400000000000004,"15min":0.54400000000000004},"min":{"1min":0.376,"5min":0.376,"15min":0.376},"max":{"1min":0.86399999999999999,"5min":0.86399999999999999,"15min":0.86399999999999999},"last":0.51600000000000001},{"interface":"front","average":{"1min":0.54800000000000004,"5min":0.54800000000000004,"15min":0.54800000000000004},"min":{"1min":0.36899999999999999,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":0.81499999999999995,"5min":0.81499999999999995,"15min":0.81499999999999995},"last":0.58799999999999997}]},{"osd":3,"last update":"Sat Sep 21 04:07:34 2024","interfaces":[{"interface":"back","average":{"1min":0.52600000000000002,"5min":0.52600000000000002,"15min":0.52600000000000002},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":0.749,"5min":0.749,"15min":0.749},"last":0.54800000000000004},{"interface":"front","average":{"1min":0.58799999999999997,"5min":0.58799999999999997,"15min":0.58799999999999997},"min":{"1min":0.39100000000000001,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":0.91900000000000004,"5min":0.91900000000000004,"15min":0.91900000000000004},"last":0.60599999999999998}]},{"osd":4,"last update":"Sat Sep 21 04:07:34 2024","interfaces":[{"interface":"back","average":{"1min":0.41199999999999998,"5min":0.41199999999999998,"15min":0.41199999999999998},"min":{"1min":0.26700000000000002,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":0.71099999999999997,"5min":0.71099999999999997,"15min":0.71099999999999997},"last":0.36099999999999999},{"interface":"front","average":{"1min":0.39900000000000002,"5min":0.39900000000000002,"15min":0.39900000000000002},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":0.70199999999999996,"5min":0.70199999999999996,"15min":0.70199999999999996},"last":0.41799999999999998}]},{"osd":6,"last update":"Sat Sep 21 04:07:58 2024","interfaces":[{"interface":"back","average":{"1min":0.47099999999999997,"5min":0.47099999999999997,"15min":0.47099999999999997},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":0.71399999999999997,"5min":0.71399999999999997,"15min":0.71399999999999997},"last":0.375},{"interface":"front","average":{"1min":0.496,"5min":0.496,"15min":0.496},"min":{"1min":0.312,"5min":0.312,"15min":0.312},"max":{"1min":0.78000000000000003,"5min":0.78000000000000003,"15min":0.78000000000000003},"last":0.436}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.53200000000000003}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-21T04:08:20.069 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 1b9b79fc-77ce-11ef-baf5-efdc52797490 -- ceph pg dump --format=json 2024-09-21T04:08:20.276 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/1b9b79fc-77ce-11ef-baf5-efdc52797490/mon.smithi089/config 2024-09-21T04:08:20.507 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:08:20 smithi089 ceph-mon[23733]: pgmap v201: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-21T04:08:20.508 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:08:20 smithi089 ceph-mon[23733]: from='client.14464 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T04:08:20.595 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:08:20 smithi195 ceph-mon[25840]: pgmap v201: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-21T04:08:20.595 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:08:20 smithi195 ceph-mon[25840]: from='client.14464 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T04:08:22.075 INFO:teuthology.orchestra.run.smithi089.stdout: 2024-09-21T04:08:22.076 INFO:teuthology.orchestra.run.smithi089.stderr:dumped all 2024-09-21T04:08:22.514 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:08:22 smithi089 ceph-mon[23733]: pgmap v202: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-21T04:08:22.514 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:08:22 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T04:08:22.514 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:08:22 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T04:08:22.515 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:08:22 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T04:08:22.515 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:08:22 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' 2024-09-21T04:08:22.608 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:08:22 smithi195 ceph-mon[25840]: pgmap v202: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-21T04:08:22.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:08:22 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T04:08:22.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:08:22 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T04:08:22.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:08:22 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T04:08:22.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:08:22 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' 2024-09-21T04:08:22.733 INFO:teuthology.orchestra.run.smithi089.stdout:{"pg_ready":true,"pg_map":{"version":202,"stamp":"2024-09-21T04:08:21.119391+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":117,"ondisk_log_size":117,"up":3,"acting":3,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":3,"num_osds":8,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":749961216,"kb_used":48928,"kb_used_data":6368,"kb_used_omap":0,"kb_used_meta":42496,"kb_avail":749912288,"statfs":{"total":767960285184,"available":767910182912,"internally_reserved":0,"allocated":6520832,"data_stored":3571206,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":43515904},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002090"},"pg_stats":[{"pgid":"1.0","version":"23'117","reported_seq":226,"reported_epoch":60,"state":"active+clean","last_fresh":"2024-09-21T04:07:24.122732+0000","last_change":"2024-09-21T04:07:24.122732+0000","last_active":"2024-09-21T04:07:24.122732+0000","last_peered":"2024-09-21T04:07:24.122732+0000","last_clean":"2024-09-21T04:07:24.122732+0000","last_became_active":"2024-09-21T04:07:24.122284+0000","last_became_peered":"2024-09-21T04:07:24.122284+0000","last_unstale":"2024-09-21T04:07:24.122732+0000","last_undegraded":"2024-09-21T04:07:24.122732+0000","last_fullsized":"2024-09-21T04:07:24.122732+0000","mapping_epoch":59,"log_start":"0'0","ondisk_log_start":"0'0","created":21,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T04:05:19.905898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T04:05:19.905898+0000","last_clean_scrub_stamp":"2024-09-21T04:05:19.905898+0000","objects_scrubbed":0,"log_size":117,"ondisk_log_size":117,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T04:17:08.405355+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]}],"pool_stats":[{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":3727360,"data_stored":3702496,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":117,"ondisk_log_size":117,"up":3,"acting":3,"num_store_stats":7}],"osd_stats":[{"osd":7,"up_from":56,"seq":240518168592,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6056,"kb_used_data":1120,"kb_used_omap":0,"kb_used_meta":4928,"kb_avail":93739096,"statfs":{"total":95995035648,"available":95988834304,"internally_reserved":0,"allocated":1146880,"data_stored":776872,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5046272},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.36199999999999999}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.442}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.35199999999999998}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.39400000000000002}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.47299999999999998}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.52600000000000002}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.503}]}]},{"osd":6,"up_from":48,"seq":206158430228,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6180,"kb_used_data":1116,"kb_used_omap":0,"kb_used_meta":5056,"kb_avail":93738972,"statfs":{"total":95995035648,"available":95988707328,"internally_reserved":0,"allocated":1142784,"data_stored":776607,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5177344},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 21 04:07:58 2024","interfaces":[{"interface":"back","average":{"1min":1.613,"5min":1.613,"15min":1.613},"min":{"1min":0.41499999999999998,"5min":0.41499999999999998,"15min":0.41499999999999998},"max":{"1min":14.903,"5min":14.903,"15min":14.903},"last":0.64000000000000001},{"interface":"front","average":{"1min":1.635,"5min":1.635,"15min":1.635},"min":{"1min":0.40699999999999997,"5min":0.40699999999999997,"15min":0.40699999999999997},"max":{"1min":14.923999999999999,"5min":14.923999999999999,"15min":14.923999999999999},"last":0.56599999999999995}]},{"osd":1,"last update":"Sat Sep 21 04:07:58 2024","interfaces":[{"interface":"back","average":{"1min":1.294,"5min":1.294,"15min":1.294},"min":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":14.885,"5min":14.885,"15min":14.885},"last":0.622},{"interface":"front","average":{"1min":0.54200000000000004,"5min":0.54200000000000004,"15min":0.54200000000000004},"min":{"1min":0.38300000000000001,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":0.83599999999999997,"5min":0.83599999999999997,"15min":0.83599999999999997},"last":0.53800000000000003}]},{"osd":2,"last update":"Sat Sep 21 04:07:58 2024","interfaces":[{"interface":"back","average":{"1min":0.53400000000000003,"5min":0.53400000000000003,"15min":0.53400000000000003},"min":{"1min":0.40000000000000002,"5min":0.40000000000000002,"15min":0.40000000000000002},"max":{"1min":0.78300000000000003,"5min":0.78300000000000003,"15min":0.78300000000000003},"last":0.60799999999999998},{"interface":"front","average":{"1min":0.56799999999999995,"5min":0.56799999999999995,"15min":0.56799999999999995},"min":{"1min":0.38900000000000001,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":0.76900000000000002,"5min":0.76900000000000002,"15min":0.76900000000000002},"last":0.58699999999999997}]},{"osd":3,"last update":"Sat Sep 21 04:07:58 2024","interfaces":[{"interface":"back","average":{"1min":0.95799999999999996,"5min":0.95799999999999996,"15min":0.95799999999999996},"min":{"1min":0.437,"5min":0.437,"15min":0.437},"max":{"1min":8.1159999999999997,"5min":8.1159999999999997,"15min":8.1159999999999997},"last":0.59799999999999998},{"interface":"front","average":{"1min":0.55900000000000005,"5min":0.55900000000000005,"15min":0.55900000000000005},"min":{"1min":0.374,"5min":0.374,"15min":0.374},"max":{"1min":0.746,"5min":0.746,"15min":0.746},"last":0.57599999999999996}]},{"osd":4,"last update":"Sat Sep 21 04:07:58 2024","interfaces":[{"interface":"back","average":{"1min":0.432,"5min":0.432,"15min":0.432},"min":{"1min":0.27500000000000002,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":0.70999999999999996,"5min":0.70999999999999996,"15min":0.70999999999999996},"last":0.42499999999999999},{"interface":"front","average":{"1min":0.47299999999999998,"5min":0.47299999999999998,"15min":0.47299999999999998},"min":{"1min":0.26600000000000001,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":0.80400000000000005,"5min":0.80400000000000005,"15min":0.80400000000000005},"last":0.47899999999999998}]},{"osd":5,"last update":"Sat Sep 21 04:07:58 2024","interfaces":[{"interface":"back","average":{"1min":0.46800000000000003,"5min":0.46800000000000003,"15min":0.46800000000000003},"min":{"1min":0.31,"5min":0.31,"15min":0.31},"max":{"1min":0.72599999999999998,"5min":0.72599999999999998,"15min":0.72599999999999998},"last":0.51100000000000001},{"interface":"front","average":{"1min":0.48899999999999999,"5min":0.48899999999999999,"15min":0.48899999999999999},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":0.79200000000000004,"5min":0.79200000000000004,"15min":0.79200000000000004},"last":0.52700000000000002}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.55200000000000005}]}]},{"osd":1,"up_from":14,"seq":60129542188,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":6372,"kb_used_data":604,"kb_used_omap":0,"kb_used_meta":5760,"kb_avail":93738780,"statfs":{"total":95995035648,"available":95988510720,"internally_reserved":0,"allocated":618496,"data_stored":248267,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5898240},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 21 04:07:55 2024","interfaces":[{"interface":"back","average":{"1min":0.34699999999999998,"5min":0.33700000000000002,"15min":0.32500000000000001},"min":{"1min":0.22900000000000001,"5min":0.20399999999999999,"15min":0.20399999999999999},"max":{"1min":0.503,"5min":0.60699999999999998,"15min":0.60699999999999998},"last":0.28499999999999998},{"interface":"front","average":{"1min":0.35299999999999998,"5min":0.34300000000000003,"15min":0.32300000000000001},"min":{"1min":0.24099999999999999,"5min":0.193,"15min":0.193},"max":{"1min":0.60199999999999998,"5min":0.64000000000000001,"15min":0.64000000000000001},"last":0.33700000000000002}]},{"osd":2,"last update":"Sat Sep 21 04:07:24 2024","interfaces":[{"interface":"back","average":{"1min":0.40999999999999998,"5min":0.35099999999999998,"15min":0.34100000000000003},"min":{"1min":0.27300000000000002,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":0.53900000000000003,"5min":0.53900000000000003,"15min":0.53900000000000003},"last":0.35099999999999998},{"interface":"front","average":{"1min":0.41099999999999998,"5min":0.36199999999999999,"15min":0.35399999999999998},"min":{"1min":0.30499999999999999,"5min":0.25,"15min":0.25},"max":{"1min":0.52700000000000002,"5min":0.58899999999999997,"15min":0.58899999999999997},"last":0.26800000000000002}]},{"osd":3,"last update":"Sat Sep 21 04:07:55 2024","interfaces":[{"interface":"back","average":{"1min":0.42399999999999999,"5min":0.45100000000000001,"15min":0.45500000000000002},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":0.64900000000000002,"5min":0.65900000000000003,"15min":0.65900000000000003},"last":0.30499999999999999},{"interface":"front","average":{"1min":0.40600000000000003,"5min":0.42499999999999999,"15min":0.42799999999999999},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":0.55100000000000005,"5min":0.63300000000000001,"15min":0.63300000000000001},"last":0.314}]},{"osd":4,"last update":"Sat Sep 21 04:08:13 2024","interfaces":[{"interface":"back","average":{"1min":0.56699999999999995,"5min":0.58799999999999997,"15min":0.59199999999999997},"min":{"1min":0.42799999999999999,"5min":0.41999999999999998,"15min":0.41999999999999998},"max":{"1min":0.70999999999999996,"5min":0.78500000000000003,"15min":0.78500000000000003},"last":0.60599999999999998},{"interface":"front","average":{"1min":0.58599999999999997,"5min":0.56499999999999995,"15min":0.56100000000000005},"min":{"1min":0.42799999999999999,"5min":0.42799999999999999,"15min":0.42799999999999999},"max":{"1min":0.79200000000000004,"5min":0.79200000000000004,"15min":0.79200000000000004},"last":0.54100000000000004}]},{"osd":5,"last update":"Sat Sep 21 04:07:33 2024","interfaces":[{"interface":"back","average":{"1min":0.56399999999999995,"5min":0.56399999999999995,"15min":0.56399999999999995},"min":{"1min":0.433,"5min":0.433,"15min":0.433},"max":{"1min":0.80900000000000005,"5min":0.80900000000000005,"15min":0.80900000000000005},"last":0.59199999999999997},{"interface":"front","average":{"1min":0.58399999999999996,"5min":0.58399999999999996,"15min":0.58399999999999996},"min":{"1min":0.45000000000000001,"5min":0.45000000000000001,"15min":0.45000000000000001},"max":{"1min":0.71499999999999997,"5min":0.71499999999999997,"15min":0.71499999999999997},"last":0.52400000000000002}]},{"osd":6,"last update":"Sat Sep 21 04:07:56 2024","interfaces":[{"interface":"back","average":{"1min":0.58999999999999997,"5min":0.58999999999999997,"15min":0.58999999999999997},"min":{"1min":0.46300000000000002,"5min":0.46300000000000002,"15min":0.46300000000000002},"max":{"1min":0.74099999999999999,"5min":0.74099999999999999,"15min":0.74099999999999999},"last":0.57099999999999995},{"interface":"front","average":{"1min":0.59999999999999998,"5min":0.59999999999999998,"15min":0.59999999999999998},"min":{"1min":0.495,"5min":0.495,"15min":0.495},"max":{"1min":0.85099999999999998,"5min":0.85099999999999998,"15min":0.85099999999999998},"last":0.56299999999999994}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.54600000000000004}]}]},{"osd":0,"up_from":9,"seq":38654705715,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6956,"kb_used_data":1124,"kb_used_omap":0,"kb_used_meta":5824,"kb_avail":93738196,"statfs":{"total":95995035648,"available":95987912704,"internally_reserved":0,"allocated":1150976,"data_stored":777195,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5963776},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 21 04:08:02 2024","interfaces":[{"interface":"back","average":{"1min":0.34200000000000003,"5min":0.34100000000000003,"15min":0.33400000000000002},"min":{"1min":0.189,"5min":0.189,"15min":0.189},"max":{"1min":0.47199999999999998,"5min":0.48399999999999999,"15min":0.48399999999999999},"last":0.28899999999999998},{"interface":"front","average":{"1min":0.33900000000000002,"5min":0.33300000000000002,"15min":0.32400000000000001},"min":{"1min":0.22,"5min":0.22,"15min":0.22},"max":{"1min":0.52500000000000002,"5min":0.52500000000000002,"15min":0.52500000000000002},"last":0.32300000000000001}]},{"osd":2,"last update":"Sat Sep 21 04:07:24 2024","interfaces":[{"interface":"back","average":{"1min":0.40999999999999998,"5min":0.35999999999999999,"15min":0.35199999999999998},"min":{"1min":0.33500000000000002,"5min":0.217,"15min":0.217},"max":{"1min":0.52700000000000002,"5min":0.55100000000000005,"15min":0.55100000000000005},"last":0.31},{"interface":"front","average":{"1min":0.38800000000000001,"5min":0.36399999999999999,"15min":0.35999999999999999},"min":{"1min":0.248,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":0.50700000000000001,"5min":0.62,"15min":0.62},"last":0.36099999999999999}]},{"osd":3,"last update":"Sat Sep 21 04:07:47 2024","interfaces":[{"interface":"back","average":{"1min":0.372,"5min":0.39400000000000002,"15min":0.39800000000000002},"min":{"1min":0.23799999999999999,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":0.50700000000000001,"5min":0.53900000000000003,"15min":0.53900000000000003},"last":0.34999999999999998},{"interface":"front","average":{"1min":0.40799999999999997,"5min":0.43099999999999999,"15min":0.435},"min":{"1min":0.31,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":0.55700000000000005,"5min":0.57499999999999996,"15min":0.57499999999999996},"last":0.33900000000000002}]},{"osd":4,"last update":"Sat Sep 21 04:08:11 2024","interfaces":[{"interface":"back","average":{"1min":0.59599999999999997,"5min":0.56000000000000005,"15min":0.55400000000000005},"min":{"1min":0.46000000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":0.79400000000000004,"5min":0.85699999999999998,"15min":0.85699999999999998},"last":0.49099999999999999},{"interface":"front","average":{"1min":0.59999999999999998,"5min":1.21,"15min":1.3120000000000001},"min":{"1min":0.371,"5min":0.371,"15min":0.371},"max":{"1min":0.81799999999999995,"5min":14.523,"15min":14.523},"last":0.52300000000000002}]},{"osd":5,"last update":"Sat Sep 21 04:07:35 2024","interfaces":[{"interface":"back","average":{"1min":0.58799999999999997,"5min":0.58799999999999997,"15min":0.58799999999999997},"min":{"1min":0.42999999999999999,"5min":0.42999999999999999,"15min":0.42999999999999999},"max":{"1min":0.73499999999999999,"5min":0.73499999999999999,"15min":0.73499999999999999},"last":0.50700000000000001},{"interface":"front","average":{"1min":0.59199999999999997,"5min":0.59199999999999997,"15min":0.59199999999999997},"min":{"1min":0.39700000000000002,"5min":0.39700000000000002,"15min":0.39700000000000002},"max":{"1min":0.82399999999999995,"5min":0.82399999999999995,"15min":0.82399999999999995},"last":0.39500000000000002}]},{"osd":6,"last update":"Sat Sep 21 04:07:56 2024","interfaces":[{"interface":"back","average":{"1min":0.58199999999999996,"5min":0.58199999999999996,"15min":0.58199999999999996},"min":{"1min":0.44400000000000001,"5min":0.44400000000000001,"15min":0.44400000000000001},"max":{"1min":0.79800000000000004,"5min":0.79800000000000004,"15min":0.79800000000000004},"last":0.38300000000000001},{"interface":"front","average":{"1min":0.59099999999999997,"5min":0.59099999999999997,"15min":0.59099999999999997},"min":{"1min":0.439,"5min":0.439,"15min":0.439},"max":{"1min":0.78800000000000003,"5min":0.78800000000000003,"15min":0.78800000000000003},"last":0.48099999999999998}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.40600000000000003}]}]},{"osd":2,"up_from":19,"seq":81604378664,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":6240,"kb_used_data":600,"kb_used_omap":0,"kb_used_meta":5632,"kb_avail":93738912,"statfs":{"total":95995035648,"available":95988645888,"internally_reserved":0,"allocated":614400,"data_stored":248002,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5767168},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 21 04:07:23 2024","interfaces":[{"interface":"back","average":{"1min":0.40999999999999998,"5min":0.38200000000000001,"15min":0.377},"min":{"1min":0.29199999999999998,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":0.56299999999999994,"5min":0.78000000000000003,"15min":0.78000000000000003},"last":0.49299999999999999},{"interface":"front","average":{"1min":0.41599999999999998,"5min":0.38400000000000001,"15min":0.379},"min":{"1min":0.28000000000000003,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":0.55900000000000005,"5min":0.70199999999999996,"15min":0.70199999999999996},"last":0.505}]},{"osd":1,"last update":"Sat Sep 21 04:07:23 2024","interfaces":[{"interface":"back","average":{"1min":0.443,"5min":0.39300000000000002,"15min":0.38500000000000001},"min":{"1min":0.29999999999999999,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":0.56299999999999994,"5min":0.60199999999999998,"15min":0.60199999999999998},"last":0.52800000000000002},{"interface":"front","average":{"1min":0.42999999999999999,"5min":0.375,"15min":0.36599999999999999},"min":{"1min":0.32700000000000001,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":0.53500000000000003,"5min":0.53500000000000003,"15min":0.53500000000000003},"last":0.52300000000000002}]},{"osd":3,"last update":"Sat Sep 21 04:07:52 2024","interfaces":[{"interface":"back","average":{"1min":0.432,"5min":0.441,"15min":0.443},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":0.68999999999999995,"5min":0.68999999999999995,"15min":0.68999999999999995},"last":0.56399999999999995},{"interface":"front","average":{"1min":0.41899999999999998,"5min":0.45200000000000001,"15min":0.45800000000000002},"min":{"1min":0.34499999999999997,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":0.63,"5min":0.63,"15min":0.63},"last":0.47099999999999997}]},{"osd":4,"last update":"Sat Sep 21 04:08:11 2024","interfaces":[{"interface":"back","average":{"1min":0.65200000000000002,"5min":0.79200000000000004,"15min":0.81599999999999995},"min":{"1min":0.47899999999999998,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":1.2350000000000001,"5min":6.2089999999999996,"15min":6.2089999999999996},"last":0.75800000000000001},{"interface":"front","average":{"1min":0.65900000000000003,"5min":0.60899999999999999,"15min":0.60099999999999998},"min":{"1min":0.44500000000000001,"5min":0.40899999999999997,"15min":0.40899999999999997},"max":{"1min":1.246,"5min":1.246,"15min":1.246},"last":0.71799999999999997}]},{"osd":5,"last update":"Sat Sep 21 04:07:35 2024","interfaces":[{"interface":"back","average":{"1min":0.57299999999999995,"5min":0.57299999999999995,"15min":0.57299999999999995},"min":{"1min":0.41799999999999998,"5min":0.41799999999999998,"15min":0.41799999999999998},"max":{"1min":0.85699999999999998,"5min":0.85699999999999998,"15min":0.85699999999999998},"last":0.627},{"interface":"front","average":{"1min":0.59499999999999997,"5min":0.59499999999999997,"15min":0.59499999999999997},"min":{"1min":0.42199999999999999,"5min":0.42199999999999999,"15min":0.42199999999999999},"max":{"1min":0.745,"5min":0.745,"15min":0.745},"last":0.78200000000000003}]},{"osd":6,"last update":"Sat Sep 21 04:07:58 2024","interfaces":[{"interface":"back","average":{"1min":0.64600000000000002,"5min":0.64600000000000002,"15min":0.64600000000000002},"min":{"1min":0.439,"5min":0.439,"15min":0.439},"max":{"1min":1.3640000000000001,"5min":1.3640000000000001,"15min":1.3640000000000001},"last":0.82299999999999995},{"interface":"front","average":{"1min":0.628,"5min":0.628,"15min":0.628},"min":{"1min":0.434,"5min":0.434,"15min":0.434},"max":{"1min":1.1970000000000001,"5min":1.1970000000000001,"15min":1.1970000000000001},"last":0.68500000000000005}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.74099999999999999}]}]},{"osd":3,"up_from":27,"seq":115964117026,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5732,"kb_used_data":604,"kb_used_omap":0,"kb_used_meta":5120,"kb_avail":93739420,"statfs":{"total":95995035648,"available":95989166080,"internally_reserved":0,"allocated":618496,"data_stored":248267,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5242880},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 21 04:07:52 2024","interfaces":[{"interface":"back","average":{"1min":0.39700000000000002,"5min":0.40899999999999997,"15min":0.41099999999999998},"min":{"1min":0.245,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":0.56699999999999995,"5min":0.61099999999999999,"15min":0.61099999999999999},"last":0.39700000000000002},{"interface":"front","average":{"1min":0.41699999999999998,"5min":0.39000000000000001,"15min":0.38600000000000001},"min":{"1min":0.29199999999999998,"5min":0.252,"15min":0.252},"max":{"1min":0.60199999999999998,"5min":0.60199999999999998,"15min":0.60199999999999998},"last":0.32300000000000001}]},{"osd":1,"last update":"Sat Sep 21 04:07:52 2024","interfaces":[{"interface":"back","average":{"1min":0.42699999999999999,"5min":0.41899999999999998,"15min":0.41799999999999998},"min":{"1min":0.23799999999999999,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":0.67600000000000005,"5min":0.67600000000000005,"15min":0.67600000000000005},"last":0.40699999999999997},{"interface":"front","average":{"1min":0.43099999999999999,"5min":0.40300000000000002,"15min":0.39800000000000002},"min":{"1min":0.27400000000000002,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":0.59699999999999998,"5min":0.59699999999999998,"15min":0.59699999999999998},"last":0.35199999999999998}]},{"osd":2,"last update":"Sat Sep 21 04:07:52 2024","interfaces":[{"interface":"back","average":{"1min":0.45500000000000002,"5min":0.44500000000000001,"15min":0.443},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":0.63800000000000001,"5min":0.65600000000000003,"15min":0.65600000000000003},"last":0.42499999999999999},{"interface":"front","average":{"1min":0.44600000000000001,"5min":0.41999999999999998,"15min":0.41599999999999998},"min":{"1min":0.28499999999999998,"5min":0.252,"15min":0.252},"max":{"1min":0.626,"5min":0.626,"15min":0.626},"last":0.38400000000000001}]},{"osd":4,"last update":"Sat Sep 21 04:08:13 2024","interfaces":[{"interface":"back","average":{"1min":0.56100000000000005,"5min":0.58399999999999996,"15min":0.58799999999999997},"min":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":0.74399999999999999,"5min":0.91900000000000004,"15min":0.91900000000000004},"last":0.628},{"interface":"front","average":{"1min":0.59499999999999997,"5min":0.60499999999999998,"15min":0.60699999999999998},"min":{"1min":0.40699999999999997,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":0.81999999999999995,"5min":0.89000000000000001,"15min":0.89000000000000001},"last":0.58999999999999997}]},{"osd":5,"last update":"Sat Sep 21 04:07:34 2024","interfaces":[{"interface":"back","average":{"1min":0.60799999999999998,"5min":0.60799999999999998,"15min":0.60799999999999998},"min":{"1min":0.47899999999999998,"5min":0.47899999999999998,"15min":0.47899999999999998},"max":{"1min":0.71999999999999997,"5min":0.71999999999999997,"15min":0.71999999999999997},"last":0.61799999999999999},{"interface":"front","average":{"1min":0.60199999999999998,"5min":0.60199999999999998,"15min":0.60199999999999998},"min":{"1min":0.42199999999999999,"5min":0.42199999999999999,"15min":0.42199999999999999},"max":{"1min":0.90700000000000003,"5min":0.90700000000000003,"15min":0.90700000000000003},"last":0.57599999999999996}]},{"osd":6,"last update":"Sat Sep 21 04:07:58 2024","interfaces":[{"interface":"back","average":{"1min":0.623,"5min":0.623,"15min":0.623},"min":{"1min":0.39700000000000002,"5min":0.39700000000000002,"15min":0.39700000000000002},"max":{"1min":0.78000000000000003,"5min":0.78000000000000003,"15min":0.78000000000000003},"last":0.56699999999999995},{"interface":"front","average":{"1min":0.66700000000000004,"5min":0.66700000000000004,"15min":0.66700000000000004},"min":{"1min":0.42799999999999999,"5min":0.42799999999999999,"15min":0.42799999999999999},"max":{"1min":0.876,"5min":0.876,"15min":0.876},"last":0.55900000000000005}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.54600000000000004}]}]},{"osd":4,"up_from":32,"seq":137438953502,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5728,"kb_used_data":600,"kb_used_omap":0,"kb_used_meta":5120,"kb_avail":93739424,"statfs":{"total":95995035648,"available":95989170176,"internally_reserved":0,"allocated":614400,"data_stored":248002,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5242880},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 21 04:08:16 2024","interfaces":[{"interface":"back","average":{"1min":0.55000000000000004,"5min":0.51500000000000001,"15min":0.50900000000000001},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":0.77800000000000002,"5min":0.80700000000000005,"15min":0.80700000000000005},"last":0.41399999999999998},{"interface":"front","average":{"1min":0.60899999999999999,"5min":0.52500000000000002,"15min":0.51100000000000001},"min":{"1min":0.38400000000000001,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":0.79300000000000004,"5min":0.79300000000000004,"15min":0.79300000000000004},"last":0.38500000000000001}]},{"osd":1,"last update":"Sat Sep 21 04:08:16 2024","interfaces":[{"interface":"back","average":{"1min":0.56000000000000005,"5min":0.53300000000000003,"15min":0.52900000000000003},"min":{"1min":0.373,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":0.73899999999999999,"5min":0.753,"15min":0.753},"last":0.439},{"interface":"front","average":{"1min":0.54800000000000004,"5min":0.51200000000000001,"15min":0.50600000000000001},"min":{"1min":0.312,"5min":0.31,"15min":0.31},"max":{"1min":0.80900000000000005,"5min":0.80900000000000005,"15min":0.80900000000000005},"last":0.35399999999999998}]},{"osd":2,"last update":"Sat Sep 21 04:08:16 2024","interfaces":[{"interface":"back","average":{"1min":0.56699999999999995,"5min":0.53200000000000003,"15min":0.52600000000000002},"min":{"1min":0.40899999999999997,"5min":0.372,"15min":0.372},"max":{"1min":0.76300000000000001,"5min":0.76300000000000001,"15min":0.76300000000000001},"last":0.46100000000000002},{"interface":"front","average":{"1min":0.55500000000000005,"5min":0.53500000000000003,"15min":0.53100000000000003},"min":{"1min":0.34399999999999997,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":0.67400000000000004,"5min":0.72499999999999998,"15min":0.72499999999999998},"last":0.55300000000000005}]},{"osd":3,"last update":"Sat Sep 21 04:08:16 2024","interfaces":[{"interface":"back","average":{"1min":0.55100000000000005,"5min":0.53600000000000003,"15min":0.53400000000000003},"min":{"1min":0.35699999999999998,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":0.72799999999999998,"5min":0.73899999999999999,"15min":0.73899999999999999},"last":0.498},{"interface":"front","average":{"1min":0.60599999999999998,"5min":0.52600000000000002,"15min":0.51200000000000001},"min":{"1min":0.39200000000000002,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":0.81899999999999995,"5min":0.81899999999999995,"15min":0.81899999999999995},"last":0.60599999999999998}]},{"osd":5,"last update":"Sat Sep 21 04:07:35 2024","interfaces":[{"interface":"back","average":{"1min":0.42899999999999999,"5min":0.42899999999999999,"15min":0.42899999999999999},"min":{"1min":0.23000000000000001,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":0.70499999999999996,"5min":0.70499999999999996,"15min":0.70499999999999996},"last":0.50600000000000001},{"interface":"front","average":{"1min":0.45400000000000001,"5min":0.45400000000000001,"15min":0.45400000000000001},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":0.81699999999999995,"5min":0.81699999999999995,"15min":0.81699999999999995},"last":0.57799999999999996}]},{"osd":6,"last update":"Sat Sep 21 04:07:59 2024","interfaces":[{"interface":"back","average":{"1min":0.49199999999999999,"5min":0.49199999999999999,"15min":0.49199999999999999},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":0.75,"5min":0.75,"15min":0.75},"last":0.53100000000000003},{"interface":"front","average":{"1min":0.504,"5min":0.504,"15min":0.504},"min":{"1min":0.29499999999999998,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":0.751,"5min":0.751,"15min":0.751},"last":0.54100000000000004}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.59299999999999997}]}]},{"osd":5,"up_from":40,"seq":171798691865,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5664,"kb_used_data":600,"kb_used_omap":0,"kb_used_meta":5056,"kb_avail":93739488,"statfs":{"total":95995035648,"available":95989235712,"internally_reserved":0,"allocated":614400,"data_stored":247994,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5177344},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 21 04:07:34 2024","interfaces":[{"interface":"back","average":{"1min":0.53000000000000003,"5min":0.53000000000000003,"15min":0.53000000000000003},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":0.73999999999999999,"5min":0.73999999999999999,"15min":0.73999999999999999},"last":0.56499999999999995},{"interface":"front","average":{"1min":0.55500000000000005,"5min":0.55500000000000005,"15min":0.55500000000000005},"min":{"1min":0.39200000000000002,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":0.90800000000000003,"5min":0.90800000000000003,"15min":0.90800000000000003},"last":0.52500000000000002}]},{"osd":1,"last update":"Sat Sep 21 04:07:34 2024","interfaces":[{"interface":"back","average":{"1min":0.52400000000000002,"5min":0.52400000000000002,"15min":0.52400000000000002},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":0.85099999999999998,"5min":0.85099999999999998,"15min":0.85099999999999998},"last":0.623},{"interface":"front","average":{"1min":0.55700000000000005,"5min":0.55700000000000005,"15min":0.55700000000000005},"min":{"1min":0.42799999999999999,"5min":0.42799999999999999,"15min":0.42799999999999999},"max":{"1min":0.81299999999999994,"5min":0.81299999999999994,"15min":0.81299999999999994},"last":0.49399999999999999}]},{"osd":2,"last update":"Sat Sep 21 04:07:34 2024","interfaces":[{"interface":"back","average":{"1min":0.54400000000000004,"5min":0.54400000000000004,"15min":0.54400000000000004},"min":{"1min":0.376,"5min":0.376,"15min":0.376},"max":{"1min":0.86399999999999999,"5min":0.86399999999999999,"15min":0.86399999999999999},"last":0.51600000000000001},{"interface":"front","average":{"1min":0.54800000000000004,"5min":0.54800000000000004,"15min":0.54800000000000004},"min":{"1min":0.36899999999999999,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":0.81499999999999995,"5min":0.81499999999999995,"15min":0.81499999999999995},"last":0.58799999999999997}]},{"osd":3,"last update":"Sat Sep 21 04:07:34 2024","interfaces":[{"interface":"back","average":{"1min":0.52600000000000002,"5min":0.52600000000000002,"15min":0.52600000000000002},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":0.749,"5min":0.749,"15min":0.749},"last":0.54800000000000004},{"interface":"front","average":{"1min":0.58799999999999997,"5min":0.58799999999999997,"15min":0.58799999999999997},"min":{"1min":0.39100000000000001,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":0.91900000000000004,"5min":0.91900000000000004,"15min":0.91900000000000004},"last":0.60599999999999998}]},{"osd":4,"last update":"Sat Sep 21 04:07:34 2024","interfaces":[{"interface":"back","average":{"1min":0.41199999999999998,"5min":0.41199999999999998,"15min":0.41199999999999998},"min":{"1min":0.26700000000000002,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":0.71099999999999997,"5min":0.71099999999999997,"15min":0.71099999999999997},"last":0.36099999999999999},{"interface":"front","average":{"1min":0.39900000000000002,"5min":0.39900000000000002,"15min":0.39900000000000002},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":0.70199999999999996,"5min":0.70199999999999996,"15min":0.70199999999999996},"last":0.41799999999999998}]},{"osd":6,"last update":"Sat Sep 21 04:07:58 2024","interfaces":[{"interface":"back","average":{"1min":0.47099999999999997,"5min":0.47099999999999997,"15min":0.47099999999999997},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":0.71399999999999997,"5min":0.71399999999999997,"15min":0.71399999999999997},"last":0.375},{"interface":"front","average":{"1min":0.496,"5min":0.496,"15min":0.496},"min":{"1min":0.312,"5min":0.312,"15min":0.312},"max":{"1min":0.78000000000000003,"5min":0.78000000000000003,"15min":0.78000000000000003},"last":0.436}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.53200000000000003}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-21T04:08:22.735 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2024-09-21T04:08:22.735 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2024-09-21T04:08:22.735 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy 2024-09-21T04:08:22.735 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 1b9b79fc-77ce-11ef-baf5-efdc52797490 -- ceph health --format=json 2024-09-21T04:08:22.948 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/1b9b79fc-77ce-11ef-baf5-efdc52797490/mon.smithi089/config 2024-09-21T04:08:23.320 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:08:23 smithi089 ceph-mon[23733]: from='client.14468 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T04:08:23.608 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:08:23 smithi195 ceph-mon[25840]: from='client.14468 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T04:08:24.361 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:08:24 smithi089 ceph-mon[23733]: pgmap v203: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-21T04:08:24.608 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:08:24 smithi195 ceph-mon[25840]: pgmap v203: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-21T04:08:24.789 INFO:teuthology.orchestra.run.smithi089.stdout: 2024-09-21T04:08:24.789 INFO:teuthology.orchestra.run.smithi089.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2024-09-21T04:08:25.364 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy done 2024-09-21T04:08:25.364 INFO:tasks.cephadm:Setup complete, yielding 2024-09-21T04:08:25.365 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-09-21T04:08:25.380 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi089.front.sepia.ceph.com 2024-09-21T04:08:25.380 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 1b9b79fc-77ce-11ef-baf5-efdc52797490 -- bash -c 'ceph orch status' 2024-09-21T04:08:25.404 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:08:25 smithi089 ceph-mon[23733]: from='client.? 172.21.15.89:0/1826902382' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-09-21T04:08:25.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:08:25 smithi195 ceph-mon[25840]: from='client.? 172.21.15.89:0/1826902382' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-09-21T04:08:26.497 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:08:26 smithi089 ceph-mon[23733]: pgmap v204: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-21T04:08:26.608 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:08:26 smithi195 ceph-mon[25840]: pgmap v204: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-21T04:08:27.468 INFO:teuthology.orchestra.run.smithi089.stdout:Backend: cephadm 2024-09-21T04:08:27.469 INFO:teuthology.orchestra.run.smithi089.stdout:Available: Yes 2024-09-21T04:08:27.469 INFO:teuthology.orchestra.run.smithi089.stdout:Paused: No 2024-09-21T04:08:28.052 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 1b9b79fc-77ce-11ef-baf5-efdc52797490 -- bash -c 'ceph orch ps' 2024-09-21T04:08:28.387 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:08:28 smithi089 ceph-mon[23733]: pgmap v205: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-21T04:08:28.387 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:08:28 smithi089 ceph-mon[23733]: from='client.14476 -' entity='client.admin' cmd=[{"prefix": "orch status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T04:08:28.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:08:28 smithi195 ceph-mon[25840]: pgmap v205: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-21T04:08:28.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:08:28 smithi195 ceph-mon[25840]: from='client.14476 -' entity='client.admin' cmd=[{"prefix": "orch status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T04:08:30.118 INFO:teuthology.orchestra.run.smithi089.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-21T04:08:30.118 INFO:teuthology.orchestra.run.smithi089.stdout:alertmanager.smithi089 smithi089 *:9093,9094 running (4m) 2m ago 5m 15.4M - ba2b418f427c 703c928c9523 2024-09-21T04:08:30.118 INFO:teuthology.orchestra.run.smithi089.stdout:crash.smithi089 smithi089 running (5m) 2m ago 5m 6409k - 17.2.0 e1d6a67b021e d1e0a8969f78 2024-09-21T04:08:30.118 INFO:teuthology.orchestra.run.smithi089.stdout:crash.smithi195 smithi195 running (5m) 71s ago 5m 6400k - 17.2.0 e1d6a67b021e 7e213ff76084 2024-09-21T04:08:30.118 INFO:teuthology.orchestra.run.smithi089.stdout:grafana.smithi089 smithi089 *:3000 running (4m) 2m ago 5m 44.9M - 8.3.5 dad864ee21e9 547de193a0ee 2024-09-21T04:08:30.118 INFO:teuthology.orchestra.run.smithi089.stdout:mgr.smithi089.srlyxf smithi089 *:9283 running (6m) 2m ago 6m 448M - 17.2.0 e1d6a67b021e 410bb00e916d 2024-09-21T04:08:30.118 INFO:teuthology.orchestra.run.smithi089.stdout:mgr.smithi195.cgotzm smithi195 *:8443,9283 running (5m) 71s ago 5m 411M - 17.2.0 e1d6a67b021e f2b03f4a2ef8 2024-09-21T04:08:30.118 INFO:teuthology.orchestra.run.smithi089.stdout:mon.smithi089 smithi089 running (6m) 2m ago 6m 42.2M 2048M 17.2.0 e1d6a67b021e 1d2b3719e46e 2024-09-21T04:08:30.119 INFO:teuthology.orchestra.run.smithi089.stdout:mon.smithi195 smithi195 running (4m) 71s ago 4m 39.8M 2048M 17.2.0 e1d6a67b021e 7f86c5b5d21e 2024-09-21T04:08:30.119 INFO:teuthology.orchestra.run.smithi089.stdout:node-exporter.smithi089 smithi089 *:9100 running (5m) 2m ago 5m 20.5M - 1dbe0e931976 5b166976fc1c 2024-09-21T04:08:30.119 INFO:teuthology.orchestra.run.smithi089.stdout:node-exporter.smithi195 smithi195 *:9100 running (4m) 71s ago 4m 16.9M - 1dbe0e931976 3448624c239b 2024-09-21T04:08:30.119 INFO:teuthology.orchestra.run.smithi089.stdout:osd.0 smithi089 running (4m) 2m ago 4m 33.6M 3215M 17.2.0 e1d6a67b021e db7c0e5020e7 2024-09-21T04:08:30.119 INFO:teuthology.orchestra.run.smithi089.stdout:osd.1 smithi089 running (3m) 2m ago 3m 35.8M 3215M 17.2.0 e1d6a67b021e f9ec697d08d0 2024-09-21T04:08:30.119 INFO:teuthology.orchestra.run.smithi089.stdout:osd.2 smithi089 running (3m) 2m ago 3m 31.9M 3215M 17.2.0 e1d6a67b021e cfa000948859 2024-09-21T04:08:30.119 INFO:teuthology.orchestra.run.smithi089.stdout:osd.3 smithi089 running (2m) 2m ago 2m 25.5M 3215M 17.2.0 e1d6a67b021e 31c610ed5815 2024-09-21T04:08:30.119 INFO:teuthology.orchestra.run.smithi089.stdout:osd.4 smithi195 running (2m) 71s ago 2m 32.6M 3984M 17.2.0 e1d6a67b021e f00e18a1f220 2024-09-21T04:08:30.119 INFO:teuthology.orchestra.run.smithi089.stdout:osd.5 smithi195 running (2m) 71s ago 2m 34.2M 3984M 17.2.0 e1d6a67b021e cecfd6ff1eee 2024-09-21T04:08:30.119 INFO:teuthology.orchestra.run.smithi089.stdout:osd.6 smithi195 running (100s) 71s ago 99s 30.5M 3984M 17.2.0 e1d6a67b021e 0d7db362cc23 2024-09-21T04:08:30.119 INFO:teuthology.orchestra.run.smithi089.stdout:osd.7 smithi195 running (76s) 71s ago 76s 10.2M 3984M 17.2.0 e1d6a67b021e 0109b3fea188 2024-09-21T04:08:30.119 INFO:teuthology.orchestra.run.smithi089.stdout:prometheus.smithi089 smithi089 *:9095 running (4m) 2m ago 4m 56.4M - 514e6a882f6e 9ed6ce8f2de3 2024-09-21T04:08:30.459 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:08:30 smithi089 ceph-mon[23733]: pgmap v206: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-21T04:08:30.595 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:08:30 smithi195 ceph-mon[25840]: pgmap v206: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-21T04:08:30.767 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 1b9b79fc-77ce-11ef-baf5-efdc52797490 -- bash -c 'ceph orch ls' 2024-09-21T04:08:31.346 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:08:31 smithi089 ceph-mon[23733]: from='client.14480 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T04:08:31.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:08:31 smithi195 ceph-mon[25840]: from='client.14480 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T04:08:32.361 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:08:32 smithi089 ceph-mon[23733]: pgmap v207: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-21T04:08:32.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:08:32 smithi195 ceph-mon[25840]: pgmap v207: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-21T04:08:32.766 INFO:teuthology.orchestra.run.smithi089.stdout:NAME PORTS RUNNING REFRESHED AGE PLACEMENT 2024-09-21T04:08:32.766 INFO:teuthology.orchestra.run.smithi089.stdout:alertmanager ?:9093,9094 1/1 2m ago 6m count:1 2024-09-21T04:08:32.766 INFO:teuthology.orchestra.run.smithi089.stdout:crash 2/2 2m ago 6m * 2024-09-21T04:08:32.766 INFO:teuthology.orchestra.run.smithi089.stdout:grafana ?:3000 1/1 2m ago 6m count:1 2024-09-21T04:08:32.766 INFO:teuthology.orchestra.run.smithi089.stdout:mgr 2/2 2m ago 6m count:2 2024-09-21T04:08:32.767 INFO:teuthology.orchestra.run.smithi089.stdout:mon 2/2 2m ago 5m smithi089:172.21.15.89=smithi089;smithi195:172.21.15.195=smithi195;count:2 2024-09-21T04:08:32.767 INFO:teuthology.orchestra.run.smithi089.stdout:node-exporter ?:9100 2/2 2m ago 6m * 2024-09-21T04:08:32.767 INFO:teuthology.orchestra.run.smithi089.stdout:osd 8 2m ago - 2024-09-21T04:08:32.767 INFO:teuthology.orchestra.run.smithi089.stdout:prometheus ?:9095 1/1 2m ago 6m count:1 2024-09-21T04:08:33.490 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:08:33 smithi089 ceph-mon[23733]: from='client.14484 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T04:08:33.513 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 1b9b79fc-77ce-11ef-baf5-efdc52797490 -- bash -c 'ceph orch host ls' 2024-09-21T04:08:33.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:08:33 smithi195 ceph-mon[25840]: from='client.14484 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T04:08:34.404 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:08:34 smithi089 ceph-mon[23733]: pgmap v208: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-21T04:08:34.608 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:08:34 smithi195 ceph-mon[25840]: pgmap v208: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-21T04:08:35.405 INFO:teuthology.orchestra.run.smithi089.stdout:HOST ADDR LABELS STATUS 2024-09-21T04:08:35.405 INFO:teuthology.orchestra.run.smithi089.stdout:smithi089 172.21.15.89 2024-09-21T04:08:35.405 INFO:teuthology.orchestra.run.smithi089.stdout:smithi195 172.21.15.195 2024-09-21T04:08:35.405 INFO:teuthology.orchestra.run.smithi089.stdout:2 hosts in cluster 2024-09-21T04:08:35.994 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 1b9b79fc-77ce-11ef-baf5-efdc52797490 -- bash -c 'ceph orch device ls' 2024-09-21T04:08:36.582 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:08:36 smithi089 ceph-mon[23733]: pgmap v209: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-21T04:08:36.582 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:08:36 smithi089 ceph-mon[23733]: from='client.14488 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T04:08:36.608 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:08:36 smithi195 ceph-mon[25840]: pgmap v209: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-21T04:08:36.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:08:36 smithi195 ceph-mon[25840]: from='client.14488 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T04:08:37.978 INFO:teuthology.orchestra.run.smithi089.stdout:HOST PATH TYPE DEVICE ID SIZE AVAILABLE REJECT REASONS 2024-09-21T04:08:37.979 INFO:teuthology.orchestra.run.smithi089.stdout:smithi089 /dev/nvme0n1 ssd INTEL_SSDPEDMD400G4_PHFT62040126400BGN 400G LVM detected, locked 2024-09-21T04:08:37.979 INFO:teuthology.orchestra.run.smithi089.stdout:smithi195 /dev/nvme0n1 ssd INTEL_SSDPED1D480GA_PHMB751300FS480DGN 480G LVM detected, locked 2024-09-21T04:08:38.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:08:38 smithi195 ceph-mon[25840]: pgmap v210: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-21T04:08:38.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:08:38 smithi195 ceph-mon[25840]: from='client.14492 -' entity='client.admin' cmd=[{"prefix": "orch device ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T04:08:38.659 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:08:38 smithi089 ceph-mon[23733]: pgmap v210: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-21T04:08:38.659 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:08:38 smithi089 ceph-mon[23733]: from='client.14492 -' entity='client.admin' cmd=[{"prefix": "orch device ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T04:08:38.684 INFO:teuthology.run_tasks:Running task vip.exec... 2024-09-21T04:08:38.723 INFO:tasks.vip:Running commands on role host.a host ubuntu@smithi089.front.sepia.ceph.com 2024-09-21T04:08:38.724 DEBUG:teuthology.orchestra.run.smithi089:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c 'systemctl stop nfs-server' 2024-09-21T04:08:38.750 INFO:teuthology.orchestra.run.smithi089.stderr:+ systemctl stop nfs-server 2024-09-21T04:08:38.758 INFO:tasks.vip:Running commands on role host.b host ubuntu@smithi195.front.sepia.ceph.com 2024-09-21T04:08:38.758 DEBUG:teuthology.orchestra.run.smithi195:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c 'systemctl stop nfs-server' 2024-09-21T04:08:38.788 INFO:teuthology.orchestra.run.smithi195.stderr:+ systemctl stop nfs-server 2024-09-21T04:08:38.797 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-09-21T04:08:38.812 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi089.front.sepia.ceph.com 2024-09-21T04:08:38.812 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 1b9b79fc-77ce-11ef-baf5-efdc52797490 -- bash -c 'ceph fs volume create foofs' 2024-09-21T04:08:40.308 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:08:40 smithi089 ceph-mon[23733]: pgmap v211: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-21T04:08:40.595 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:08:40 smithi195 ceph-mon[25840]: pgmap v211: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-21T04:08:41.608 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:08:41 smithi195 ceph-mon[25840]: from='client.14496 -' entity='client.admin' cmd=[{"prefix": "fs volume create", "name": "foofs", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T04:08:41.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:08:41 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foofs.meta"}]: dispatch 2024-09-21T04:08:41.764 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:08:41 smithi089 ceph-mon[23733]: from='client.14496 -' entity='client.admin' cmd=[{"prefix": "fs volume create", "name": "foofs", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T04:08:41.764 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:08:41 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foofs.meta"}]: dispatch 2024-09-21T04:08:42.608 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:08:42 smithi195 ceph-mon[25840]: pgmap v212: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-21T04:08:42.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:08:42 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foofs.meta"}]': finished 2024-09-21T04:08:42.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:08:42 smithi195 ceph-mon[25840]: osdmap e61: 8 total, 8 up, 8 in 2024-09-21T04:08:42.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:08:42 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foofs.data"}]: dispatch 2024-09-21T04:08:42.642 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:08:42 smithi089 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-mon-smithi089[23729]: 2024-09-21T04:08:42.283+0000 7fa3af93d700 -1 log_channel(cluster) log [ERR] : Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-09-21T04:08:42.642 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:08:42 smithi089 ceph-mon[23733]: pgmap v212: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-21T04:08:42.642 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:08:42 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foofs.meta"}]': finished 2024-09-21T04:08:42.642 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:08:42 smithi089 ceph-mon[23733]: osdmap e61: 8 total, 8 up, 8 in 2024-09-21T04:08:42.642 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:08:42 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foofs.data"}]: dispatch 2024-09-21T04:08:43.042 INFO:teuthology.run_tasks:Running task cephadm.wait_for_service... 2024-09-21T04:08:43.059 INFO:tasks.cephadm:Waiting for ceph service mds.foofs to start (timeout 300)... 2024-09-21T04:08:43.059 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 1b9b79fc-77ce-11ef-baf5-efdc52797490 -- ceph orch ls -f json 2024-09-21T04:08:43.544 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:08:43 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foofs.data"}]': finished 2024-09-21T04:08:43.544 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:08:43 smithi089 ceph-mon[23733]: osdmap e62: 8 total, 8 up, 8 in 2024-09-21T04:08:43.544 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:08:43 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "fs new", "fs_name": "foofs", "metadata": "cephfs.foofs.meta", "data": "cephfs.foofs.data"}]: dispatch 2024-09-21T04:08:43.544 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:08:43 smithi089 ceph-mon[23733]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-09-21T04:08:43.544 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:08:43 smithi089 ceph-mon[23733]: Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2024-09-21T04:08:43.545 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:08:43 smithi089 ceph-mon[23733]: osdmap e63: 8 total, 8 up, 8 in 2024-09-21T04:08:43.545 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:08:43 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd='[{"prefix": "fs new", "fs_name": "foofs", "metadata": "cephfs.foofs.meta", "data": "cephfs.foofs.data"}]': finished 2024-09-21T04:08:43.545 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:08:43 smithi089 ceph-mon[23733]: fsmap foofs:0 2024-09-21T04:08:43.545 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:08:43 smithi089 ceph-mon[23733]: Saving service mds.foofs spec with placement count:2 2024-09-21T04:08:43.545 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:08:43 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' 2024-09-21T04:08:43.545 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:08:43 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T04:08:43.545 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:08:43 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T04:08:43.545 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:08:43 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T04:08:43.545 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:08:43 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' 2024-09-21T04:08:43.545 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:08:43 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' 2024-09-21T04:08:43.545 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:08:43 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi089.osufws", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-09-21T04:08:43.545 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:08:43 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi089.osufws", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-09-21T04:08:43.545 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:08:43 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T04:08:43.545 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:08:43 smithi089 ceph-mon[23733]: Deploying daemon mds.foofs.smithi089.osufws on smithi089 2024-09-21T04:08:43.545 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:08:43 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' 2024-09-21T04:08:43.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:08:43 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foofs.data"}]': finished 2024-09-21T04:08:43.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:08:43 smithi195 ceph-mon[25840]: osdmap e62: 8 total, 8 up, 8 in 2024-09-21T04:08:43.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:08:43 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "fs new", "fs_name": "foofs", "metadata": "cephfs.foofs.meta", "data": "cephfs.foofs.data"}]: dispatch 2024-09-21T04:08:43.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:08:43 smithi195 ceph-mon[25840]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-09-21T04:08:43.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:08:43 smithi195 ceph-mon[25840]: Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2024-09-21T04:08:43.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:08:43 smithi195 ceph-mon[25840]: osdmap e63: 8 total, 8 up, 8 in 2024-09-21T04:08:43.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:08:43 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd='[{"prefix": "fs new", "fs_name": "foofs", "metadata": "cephfs.foofs.meta", "data": "cephfs.foofs.data"}]': finished 2024-09-21T04:08:43.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:08:43 smithi195 ceph-mon[25840]: fsmap foofs:0 2024-09-21T04:08:43.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:08:43 smithi195 ceph-mon[25840]: Saving service mds.foofs spec with placement count:2 2024-09-21T04:08:43.610 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:08:43 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' 2024-09-21T04:08:43.610 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:08:43 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T04:08:43.610 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:08:43 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T04:08:43.610 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:08:43 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T04:08:43.610 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:08:43 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' 2024-09-21T04:08:43.610 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:08:43 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' 2024-09-21T04:08:43.610 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:08:43 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi089.osufws", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-09-21T04:08:43.610 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:08:43 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi089.osufws", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-09-21T04:08:43.610 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:08:43 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T04:08:43.610 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:08:43 smithi195 ceph-mon[25840]: Deploying daemon mds.foofs.smithi089.osufws on smithi089 2024-09-21T04:08:43.610 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:08:43 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' 2024-09-21T04:08:44.514 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:08:44 smithi089 ceph-mon[23733]: pgmap v216: 65 pgs: 13 creating+peering, 6 active+clean, 46 unknown; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-21T04:08:44.514 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:08:44 smithi089 ceph-mon[23733]: osdmap e64: 8 total, 8 up, 8 in 2024-09-21T04:08:44.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:08:44 smithi195 ceph-mon[25840]: pgmap v216: 65 pgs: 13 creating+peering, 6 active+clean, 46 unknown; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-21T04:08:44.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:08:44 smithi195 ceph-mon[25840]: osdmap e64: 8 total, 8 up, 8 in 2024-09-21T04:08:45.515 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:08:45 smithi089 ceph-mon[23733]: osdmap e65: 8 total, 8 up, 8 in 2024-09-21T04:08:45.608 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:08:45 smithi195 ceph-mon[25840]: osdmap e65: 8 total, 8 up, 8 in 2024-09-21T04:08:46.608 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:08:46 smithi195 ceph-mon[25840]: pgmap v219: 65 pgs: 13 creating+peering, 15 active+clean, 37 unknown; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-21T04:08:46.633 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:08:46 smithi089 ceph-mon[23733]: pgmap v219: 65 pgs: 13 creating+peering, 15 active+clean, 37 unknown; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-21T04:08:46.814 INFO:teuthology.orchestra.run.smithi089.stdout: 2024-09-21T04:08:46.814 INFO:teuthology.orchestra.run.smithi089.stdout:[{"placement": {"count": 1}, "service_name": "alertmanager", "service_type": "alertmanager", "status": {"created": "2024-09-21T04:02:13.825693Z", "last_refresh": "2024-09-21T04:05:54.589648Z", "ports": [9093, 9094], "running": 1, "size": 1}}, {"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"created": "2024-09-21T04:02:08.520221Z", "last_refresh": "2024-09-21T04:05:54.589711Z", "running": 2, "size": 2}}, {"placement": {"count": 1}, "service_name": "grafana", "service_type": "grafana", "status": {"created": "2024-09-21T04:02:10.882761Z", "last_refresh": "2024-09-21T04:05:54.589771Z", "ports": [3000], "running": 1, "size": 1}}, {"events": ["2024-09-21T04:08:42.306697Z service:mds.foofs [INFO] \"service was created\""], "placement": {"count": 2}, "service_id": "foofs", "service_name": "mds.foofs", "service_type": "mds", "status": {"created": "2024-09-21T04:08:42.303179Z", "running": 0, "size": 2}}, {"placement": {"count": 2}, "service_name": "mgr", "service_type": "mgr", "status": {"created": "2024-09-21T04:02:06.728724Z", "last_refresh": "2024-09-21T04:05:54.589578Z", "running": 2, "size": 2}}, {"placement": {"count": 2, "hosts": ["smithi089:172.21.15.89=smithi089", "smithi195:172.21.15.195=smithi195"]}, "service_name": "mon", "service_type": "mon", "status": {"created": "2024-09-21T04:02:55.539704Z", "last_refresh": "2024-09-21T04:05:54.589417Z", "running": 2, "size": 2}}, {"placement": {"host_pattern": "*"}, "service_name": "node-exporter", "service_type": "node-exporter", "status": {"created": "2024-09-21T04:02:12.608673Z", "last_refresh": "2024-09-21T04:05:54.589830Z", "ports": [9100], "running": 2, "size": 2}}, {"service_name": "osd", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9", "container_image_name": "quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a", "last_refresh": "2024-09-21T04:05:54.589948Z", "running": 8, "size": 8}, "unmanaged": true}, {"placement": {"count": 1}, "service_name": "prometheus", "service_type": "prometheus", "status": {"created": "2024-09-21T04:02:09.740490Z", "last_refresh": "2024-09-21T04:05:54.589889Z", "ports": [9095], "running": 1, "size": 1}}] 2024-09-21T04:08:47.427 INFO:tasks.cephadm:mds.foofs has 0/2 2024-09-21T04:08:47.764 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:08:47 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' 2024-09-21T04:08:47.765 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:08:47 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi195.hijxre", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-09-21T04:08:47.765 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:08:47 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi195.hijxre", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-09-21T04:08:47.765 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:08:47 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T04:08:47.765 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:08:47 smithi089 ceph-mon[23733]: Deploying daemon mds.foofs.smithi195.hijxre on smithi195 2024-09-21T04:08:47.765 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:08:47 smithi089 ceph-mon[23733]: from='client.14504 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T04:08:47.765 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:08:47 smithi089 ceph-mon[23733]: daemon mds.foofs.smithi089.osufws assigned to filesystem foofs as rank 0 (now has 1 ranks) 2024-09-21T04:08:47.765 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:08:47 smithi089 ceph-mon[23733]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-09-21T04:08:47.765 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:08:47 smithi089 ceph-mon[23733]: Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2024-09-21T04:08:47.765 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:08:47 smithi089 ceph-mon[23733]: Cluster is now healthy 2024-09-21T04:08:47.765 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:08:47 smithi089 ceph-mon[23733]: mds.? [v2:172.21.15.89:6834/345305443,v1:172.21.15.89:6835/345305443] up:boot 2024-09-21T04:08:47.765 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:08:47 smithi089 ceph-mon[23733]: fsmap foofs:1 {0=foofs.smithi089.osufws=up:creating} 2024-09-21T04:08:47.765 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:08:47 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi089.osufws"}]: dispatch 2024-09-21T04:08:47.766 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:08:47 smithi089 ceph-mon[23733]: daemon mds.foofs.smithi089.osufws is now active in filesystem foofs as rank 0 2024-09-21T04:08:47.859 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:08:47 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' 2024-09-21T04:08:47.859 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:08:47 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi195.hijxre", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-09-21T04:08:47.859 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:08:47 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi195.hijxre", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-09-21T04:08:47.859 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:08:47 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T04:08:47.859 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:08:47 smithi195 ceph-mon[25840]: Deploying daemon mds.foofs.smithi195.hijxre on smithi195 2024-09-21T04:08:47.859 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:08:47 smithi195 ceph-mon[25840]: from='client.14504 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T04:08:47.859 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:08:47 smithi195 ceph-mon[25840]: daemon mds.foofs.smithi089.osufws assigned to filesystem foofs as rank 0 (now has 1 ranks) 2024-09-21T04:08:47.859 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:08:47 smithi195 ceph-mon[25840]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-09-21T04:08:47.860 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:08:47 smithi195 ceph-mon[25840]: Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2024-09-21T04:08:47.860 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:08:47 smithi195 ceph-mon[25840]: Cluster is now healthy 2024-09-21T04:08:47.860 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:08:47 smithi195 ceph-mon[25840]: mds.? [v2:172.21.15.89:6834/345305443,v1:172.21.15.89:6835/345305443] up:boot 2024-09-21T04:08:47.860 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:08:47 smithi195 ceph-mon[25840]: fsmap foofs:1 {0=foofs.smithi089.osufws=up:creating} 2024-09-21T04:08:47.860 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:08:47 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi089.osufws"}]: dispatch 2024-09-21T04:08:47.860 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:08:47 smithi195 ceph-mon[25840]: daemon mds.foofs.smithi089.osufws is now active in filesystem foofs as rank 0 2024-09-21T04:08:48.428 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 1b9b79fc-77ce-11ef-baf5-efdc52797490 -- ceph orch ls -f json 2024-09-21T04:08:48.591 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:08:48 smithi195 ceph-mon[25840]: pgmap v220: 65 pgs: 5 creating+peering, 60 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2024-09-21T04:08:48.592 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:08:48 smithi195 ceph-mon[25840]: mds.? [v2:172.21.15.89:6834/345305443,v1:172.21.15.89:6835/345305443] up:active 2024-09-21T04:08:48.592 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:08:48 smithi195 ceph-mon[25840]: fsmap foofs:1 {0=foofs.smithi089.osufws=up:active} 2024-09-21T04:08:48.736 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:08:48 smithi089 ceph-mon[23733]: pgmap v220: 65 pgs: 5 creating+peering, 60 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2024-09-21T04:08:48.736 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:08:48 smithi089 ceph-mon[23733]: mds.? [v2:172.21.15.89:6834/345305443,v1:172.21.15.89:6835/345305443] up:active 2024-09-21T04:08:48.736 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:08:48 smithi089 ceph-mon[23733]: fsmap foofs:1 {0=foofs.smithi089.osufws=up:active} 2024-09-21T04:08:49.965 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:08:49 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' 2024-09-21T04:08:49.965 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:08:49 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T04:08:49.965 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:08:49 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T04:08:49.965 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:08:49 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T04:08:49.965 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:08:49 smithi089 ceph-mon[23733]: mds.? [v2:172.21.15.195:6832/3071482597,v1:172.21.15.195:6833/3071482597] up:boot 2024-09-21T04:08:49.965 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:08:49 smithi089 ceph-mon[23733]: fsmap foofs:1 {0=foofs.smithi089.osufws=up:active} 1 up:standby 2024-09-21T04:08:49.965 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:08:49 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi195.hijxre"}]: dispatch 2024-09-21T04:08:49.965 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:08:49 smithi089 ceph-mon[23733]: fsmap foofs:1 {0=foofs.smithi089.osufws=up:active} 1 up:standby 2024-09-21T04:08:50.109 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:08:49 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' 2024-09-21T04:08:50.109 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:08:49 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T04:08:50.109 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:08:49 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T04:08:50.109 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:08:49 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T04:08:50.109 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:08:49 smithi195 ceph-mon[25840]: mds.? [v2:172.21.15.195:6832/3071482597,v1:172.21.15.195:6833/3071482597] up:boot 2024-09-21T04:08:50.109 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:08:49 smithi195 ceph-mon[25840]: fsmap foofs:1 {0=foofs.smithi089.osufws=up:active} 1 up:standby 2024-09-21T04:08:50.109 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:08:49 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi195.hijxre"}]: dispatch 2024-09-21T04:08:50.109 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:08:49 smithi195 ceph-mon[25840]: fsmap foofs:1 {0=foofs.smithi089.osufws=up:active} 1 up:standby 2024-09-21T04:08:50.387 INFO:teuthology.orchestra.run.smithi089.stdout: 2024-09-21T04:08:50.387 INFO:teuthology.orchestra.run.smithi089.stdout:[{"placement": {"count": 1}, "service_name": "alertmanager", "service_type": "alertmanager", "status": {"created": "2024-09-21T04:02:13.825693Z", "last_refresh": "2024-09-21T04:05:54.589648Z", "ports": [9093, 9094], "running": 1, "size": 1}}, {"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"created": "2024-09-21T04:02:08.520221Z", "last_refresh": "2024-09-21T04:05:54.589711Z", "running": 2, "size": 2}}, {"placement": {"count": 1}, "service_name": "grafana", "service_type": "grafana", "status": {"created": "2024-09-21T04:02:10.882761Z", "last_refresh": "2024-09-21T04:05:54.589771Z", "ports": [3000], "running": 1, "size": 1}}, {"events": ["2024-09-21T04:08:42.306697Z service:mds.foofs [INFO] \"service was created\""], "placement": {"count": 2}, "service_id": "foofs", "service_name": "mds.foofs", "service_type": "mds", "status": {"created": "2024-09-21T04:08:42.303179Z", "running": 0, "size": 2}}, {"placement": {"count": 2}, "service_name": "mgr", "service_type": "mgr", "status": {"created": "2024-09-21T04:02:06.728724Z", "last_refresh": "2024-09-21T04:05:54.589578Z", "running": 2, "size": 2}}, {"placement": {"count": 2, "hosts": ["smithi089:172.21.15.89=smithi089", "smithi195:172.21.15.195=smithi195"]}, "service_name": "mon", "service_type": "mon", "status": {"created": "2024-09-21T04:02:55.539704Z", "last_refresh": "2024-09-21T04:05:54.589417Z", "running": 2, "size": 2}}, {"placement": {"host_pattern": "*"}, "service_name": "node-exporter", "service_type": "node-exporter", "status": {"created": "2024-09-21T04:02:12.608673Z", "last_refresh": "2024-09-21T04:05:54.589830Z", "ports": [9100], "running": 2, "size": 2}}, {"service_name": "osd", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9", "container_image_name": "quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a", "last_refresh": "2024-09-21T04:05:54.589948Z", "running": 8, "size": 8}, "unmanaged": true}, {"placement": {"count": 1}, "service_name": "prometheus", "service_type": "prometheus", "status": {"created": "2024-09-21T04:02:09.740490Z", "last_refresh": "2024-09-21T04:05:54.589889Z", "ports": [9095], "running": 1, "size": 1}}] 2024-09-21T04:08:51.014 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:08:50 smithi089 ceph-mon[23733]: pgmap v221: 65 pgs: 65 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 0 B/s wr, 2 op/s 2024-09-21T04:08:51.014 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:08:50 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' 2024-09-21T04:08:51.108 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:08:50 smithi195 ceph-mon[25840]: pgmap v221: 65 pgs: 65 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 0 B/s wr, 2 op/s 2024-09-21T04:08:51.109 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:08:50 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' 2024-09-21T04:08:51.472 INFO:tasks.cephadm:mds.foofs has 0/2 2024-09-21T04:08:52.108 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:08:51 smithi195 ceph-mon[25840]: from='client.14510 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T04:08:52.141 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:08:51 smithi089 ceph-mon[23733]: from='client.14510 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T04:08:52.473 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 1b9b79fc-77ce-11ef-baf5-efdc52797490 -- ceph orch ls -f json 2024-09-21T04:08:53.108 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:08:52 smithi195 ceph-mon[25840]: pgmap v222: 65 pgs: 65 active+clean; 577 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1023 B/s wr, 3 op/s 2024-09-21T04:08:53.121 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:08:52 smithi089 ceph-mon[23733]: pgmap v222: 65 pgs: 65 active+clean; 577 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1023 B/s wr, 3 op/s 2024-09-21T04:08:54.264 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:08:54 smithi089 ceph-mon[23733]: pgmap v223: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s wr, 4 op/s 2024-09-21T04:08:54.265 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:08:54 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' 2024-09-21T04:08:54.608 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:08:54 smithi195 ceph-mon[25840]: pgmap v223: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s wr, 4 op/s 2024-09-21T04:08:54.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:08:54 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' 2024-09-21T04:08:55.831 INFO:teuthology.orchestra.run.smithi089.stdout: 2024-09-21T04:08:55.832 INFO:teuthology.orchestra.run.smithi089.stdout:[{"placement": {"count": 1}, "service_name": "alertmanager", "service_type": "alertmanager", "status": {"created": "2024-09-21T04:02:13.825693Z", "last_refresh": "2024-09-21T04:05:54.589648Z", "ports": [9093, 9094], "running": 1, "size": 1}}, {"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"created": "2024-09-21T04:02:08.520221Z", "last_refresh": "2024-09-21T04:05:54.589711Z", "running": 2, "size": 2}}, {"placement": {"count": 1}, "service_name": "grafana", "service_type": "grafana", "status": {"created": "2024-09-21T04:02:10.882761Z", "last_refresh": "2024-09-21T04:05:54.589771Z", "ports": [3000], "running": 1, "size": 1}}, {"events": ["2024-09-21T04:08:42.306697Z service:mds.foofs [INFO] \"service was created\""], "placement": {"count": 2}, "service_id": "foofs", "service_name": "mds.foofs", "service_type": "mds", "status": {"created": "2024-09-21T04:08:42.303179Z", "last_refresh": "2024-09-21T04:08:50.771208Z", "running": 1, "size": 2}}, {"placement": {"count": 2}, "service_name": "mgr", "service_type": "mgr", "status": {"created": "2024-09-21T04:02:06.728724Z", "last_refresh": "2024-09-21T04:05:54.589578Z", "running": 2, "size": 2}}, {"placement": {"count": 2, "hosts": ["smithi089:172.21.15.89=smithi089", "smithi195:172.21.15.195=smithi195"]}, "service_name": "mon", "service_type": "mon", "status": {"created": "2024-09-21T04:02:55.539704Z", "last_refresh": "2024-09-21T04:05:54.589417Z", "running": 2, "size": 2}}, {"placement": {"host_pattern": "*"}, "service_name": "node-exporter", "service_type": "node-exporter", "status": {"created": "2024-09-21T04:02:12.608673Z", "last_refresh": "2024-09-21T04:05:54.589830Z", "ports": [9100], "running": 2, "size": 2}}, {"service_name": "osd", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9", "container_image_name": "quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a", "last_refresh": "2024-09-21T04:05:54.589948Z", "running": 8, "size": 8}, "unmanaged": true}, {"placement": {"count": 1}, "service_name": "prometheus", "service_type": "prometheus", "status": {"created": "2024-09-21T04:02:09.740490Z", "last_refresh": "2024-09-21T04:05:54.589889Z", "ports": [9095], "running": 1, "size": 1}}] 2024-09-21T04:08:56.514 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:08:56 smithi089 ceph-mon[23733]: pgmap v224: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s wr, 4 op/s 2024-09-21T04:08:56.514 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:08:56 smithi089 ceph-mon[23733]: from='client.14514 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T04:08:56.608 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:08:56 smithi195 ceph-mon[25840]: pgmap v224: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s wr, 4 op/s 2024-09-21T04:08:56.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:08:56 smithi195 ceph-mon[25840]: from='client.14514 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T04:08:56.639 INFO:tasks.cephadm:mds.foofs has 1/2 2024-09-21T04:08:57.640 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 1b9b79fc-77ce-11ef-baf5-efdc52797490 -- ceph orch ls -f json 2024-09-21T04:08:57.942 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:08:57 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' 2024-09-21T04:08:57.942 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:08:57 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' 2024-09-21T04:08:57.942 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:08:57 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T04:08:57.942 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:08:57 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T04:08:57.942 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:08:57 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T04:08:57.942 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:08:57 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' 2024-09-21T04:08:58.358 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:08:57 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' 2024-09-21T04:08:58.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:08:57 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' 2024-09-21T04:08:58.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:08:57 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T04:08:58.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:08:57 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T04:08:58.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:08:57 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T04:08:58.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:08:57 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' 2024-09-21T04:08:59.248 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:08:58 smithi089 ceph-mon[23733]: pgmap v225: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-09-21T04:08:59.249 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:08:58 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' 2024-09-21T04:08:59.249 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:08:58 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' 2024-09-21T04:08:59.358 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:08:58 smithi195 ceph-mon[25840]: pgmap v225: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-09-21T04:08:59.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:08:58 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' 2024-09-21T04:08:59.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:08:58 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' 2024-09-21T04:08:59.729 INFO:teuthology.orchestra.run.smithi089.stdout: 2024-09-21T04:08:59.729 INFO:teuthology.orchestra.run.smithi089.stdout:[{"placement": {"count": 1}, "service_name": "alertmanager", "service_type": "alertmanager", "status": {"created": "2024-09-21T04:02:13.825693Z", "last_refresh": "2024-09-21T04:08:56.891538Z", "ports": [9093, 9094], "running": 1, "size": 1}}, {"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"created": "2024-09-21T04:02:08.520221Z", "last_refresh": "2024-09-21T04:08:50.770609Z", "running": 2, "size": 2}}, {"placement": {"count": 1}, "service_name": "grafana", "service_type": "grafana", "status": {"created": "2024-09-21T04:02:10.882761Z", "last_refresh": "2024-09-21T04:08:56.891667Z", "ports": [3000], "running": 1, "size": 1}}, {"events": ["2024-09-21T04:08:42.306697Z service:mds.foofs [INFO] \"service was created\""], "placement": {"count": 2}, "service_id": "foofs", "service_name": "mds.foofs", "service_type": "mds", "status": {"created": "2024-09-21T04:08:42.303179Z", "last_refresh": "2024-09-21T04:08:50.771208Z", "running": 2, "size": 2}}, {"placement": {"count": 2}, "service_name": "mgr", "service_type": "mgr", "status": {"created": "2024-09-21T04:02:06.728724Z", "last_refresh": "2024-09-21T04:08:50.770739Z", "running": 2, "size": 2}}, {"placement": {"count": 2, "hosts": ["smithi089:172.21.15.89=smithi089", "smithi195:172.21.15.195=smithi195"]}, "service_name": "mon", "service_type": "mon", "status": {"created": "2024-09-21T04:02:55.539704Z", "last_refresh": "2024-09-21T04:08:50.770808Z", "running": 2, "size": 2}}, {"placement": {"host_pattern": "*"}, "service_name": "node-exporter", "service_type": "node-exporter", "status": {"created": "2024-09-21T04:02:12.608673Z", "last_refresh": "2024-09-21T04:08:50.770870Z", "ports": [9100], "running": 2, "size": 2}}, {"service_name": "osd", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9", "container_image_name": "quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a", "last_refresh": "2024-09-21T04:08:50.770931Z", "running": 8, "size": 8}, "unmanaged": true}, {"placement": {"count": 1}, "service_name": "prometheus", "service_type": "prometheus", "status": {"created": "2024-09-21T04:02:09.740490Z", "last_refresh": "2024-09-21T04:08:56.891792Z", "ports": [9095], "running": 1, "size": 1}}] 2024-09-21T04:09:00.373 INFO:tasks.cephadm:mds.foofs has 2/2 2024-09-21T04:09:00.373 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-09-21T04:09:00.382 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi089.front.sepia.ceph.com 2024-09-21T04:09:00.383 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 1b9b79fc-77ce-11ef-baf5-efdc52797490 -- bash -c 'ceph nfs cluster create foo --placement=2 || ceph nfs cluster create cephfs foo --placement=2' 2024-09-21T04:09:00.945 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:09:00 smithi089 ceph-mon[23733]: pgmap v226: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-09-21T04:09:00.945 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:09:00 smithi089 ceph-mon[23733]: from='client.14518 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T04:09:01.358 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:09:00 smithi195 ceph-mon[25840]: pgmap v226: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-09-21T04:09:01.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:09:00 smithi195 ceph-mon[25840]: from='client.14518 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T04:09:03.264 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:09:02 smithi089 ceph-mon[23733]: pgmap v227: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 2 op/s 2024-09-21T04:09:03.265 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:09:02 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "osd pool create", "pool": ".nfs"}]: dispatch 2024-09-21T04:09:03.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:09:02 smithi195 ceph-mon[25840]: pgmap v227: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 2 op/s 2024-09-21T04:09:03.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:09:02 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "osd pool create", "pool": ".nfs"}]: dispatch 2024-09-21T04:09:04.171 INFO:teuthology.orchestra.run.smithi089.stdout:NFS Cluster Created Successfully 2024-09-21T04:09:04.186 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:09:03 smithi089 ceph-mon[23733]: from='client.14522 -' entity='client.admin' cmd=[{"prefix": "nfs cluster create", "cluster_id": "foo", "placement": "2", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T04:09:04.186 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:09:03 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd='[{"prefix": "osd pool create", "pool": ".nfs"}]': finished 2024-09-21T04:09:04.186 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:09:03 smithi089 ceph-mon[23733]: osdmap e66: 8 total, 8 up, 8 in 2024-09-21T04:09:04.186 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:09:03 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "osd pool application enable", "pool": ".nfs", "app": "nfs"}]: dispatch 2024-09-21T04:09:04.260 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:09:03 smithi195 ceph-mon[25840]: from='client.14522 -' entity='client.admin' cmd=[{"prefix": "nfs cluster create", "cluster_id": "foo", "placement": "2", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T04:09:04.260 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:09:03 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd='[{"prefix": "osd pool create", "pool": ".nfs"}]': finished 2024-09-21T04:09:04.260 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:09:03 smithi195 ceph-mon[25840]: osdmap e66: 8 total, 8 up, 8 in 2024-09-21T04:09:04.260 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:09:03 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "osd pool application enable", "pool": ".nfs", "app": "nfs"}]: dispatch 2024-09-21T04:09:04.898 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 1b9b79fc-77ce-11ef-baf5-efdc52797490 -- bash -c 'ceph nfs export create cephfs --fsname foofs --clusterid foo --binding /fake || ceph nfs export create cephfs --fsname foofs --cluster-id foo --pseudo-path /fake' 2024-09-21T04:09:05.226 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:09:04 smithi089 ceph-mon[23733]: pgmap v228: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 426 B/s wr, 1 op/s 2024-09-21T04:09:05.226 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:09:04 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd='[{"prefix": "osd pool application enable", "pool": ".nfs", "app": "nfs"}]': finished 2024-09-21T04:09:05.226 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:09:04 smithi089 ceph-mon[23733]: osdmap e67: 8 total, 8 up, 8 in 2024-09-21T04:09:05.227 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:09:04 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' 2024-09-21T04:09:05.227 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:09:04 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T04:09:05.227 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:09:04 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T04:09:05.227 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:09:04 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T04:09:05.227 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:09:04 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' 2024-09-21T04:09:05.227 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:09:04 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' 2024-09-21T04:09:05.227 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:09:04 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi089.dcjmqc", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]: dispatch 2024-09-21T04:09:05.227 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:09:04 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi089.dcjmqc", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]': finished 2024-09-21T04:09:05.227 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:09:04 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]: dispatch 2024-09-21T04:09:05.227 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:09:04 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-09-21T04:09:05.227 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:09:04 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T04:09:05.227 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:09:04 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]: dispatch 2024-09-21T04:09:05.227 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:09:04 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-09-21T04:09:05.227 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:09:04 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi089.dcjmqc-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-09-21T04:09:05.228 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:09:04 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi089.dcjmqc-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-09-21T04:09:05.228 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:09:04 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T04:09:05.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:09:04 smithi195 ceph-mon[25840]: pgmap v228: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 426 B/s wr, 1 op/s 2024-09-21T04:09:05.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:09:04 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd='[{"prefix": "osd pool application enable", "pool": ".nfs", "app": "nfs"}]': finished 2024-09-21T04:09:05.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:09:04 smithi195 ceph-mon[25840]: osdmap e67: 8 total, 8 up, 8 in 2024-09-21T04:09:05.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:09:04 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' 2024-09-21T04:09:05.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:09:04 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T04:09:05.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:09:04 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T04:09:05.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:09:04 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T04:09:05.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:09:04 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' 2024-09-21T04:09:05.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:09:04 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' 2024-09-21T04:09:05.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:09:04 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi089.dcjmqc", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]: dispatch 2024-09-21T04:09:05.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:09:04 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi089.dcjmqc", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]': finished 2024-09-21T04:09:05.360 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:09:04 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]: dispatch 2024-09-21T04:09:05.360 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:09:04 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-09-21T04:09:05.360 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:09:04 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T04:09:05.360 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:09:04 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]: dispatch 2024-09-21T04:09:05.360 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:09:04 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-09-21T04:09:05.360 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:09:04 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi089.dcjmqc-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-09-21T04:09:05.360 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:09:04 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi089.dcjmqc-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-09-21T04:09:05.360 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:09:04 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T04:09:06.208 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:09:05 smithi089 ceph-mon[23733]: Saving service nfs.foo spec with placement count:2 2024-09-21T04:09:06.208 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:09:05 smithi089 ceph-mon[23733]: Creating key for client.nfs.foo.0.0.smithi089.dcjmqc 2024-09-21T04:09:06.208 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:09:05 smithi089 ceph-mon[23733]: Ensuring nfs.foo.0 is in the ganesha grace table 2024-09-21T04:09:06.208 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:09:05 smithi089 ceph-mon[23733]: Rados config object exists: conf-nfs.foo 2024-09-21T04:09:06.208 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:09:05 smithi089 ceph-mon[23733]: Creating key for client.nfs.foo.0.0.smithi089.dcjmqc-rgw 2024-09-21T04:09:06.208 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:09:05 smithi089 ceph-mon[23733]: Deploying daemon nfs.foo.0.0.smithi089.dcjmqc on smithi089 2024-09-21T04:09:06.208 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:09:05 smithi089 ceph-mon[23733]: osdmap e68: 8 total, 8 up, 8 in 2024-09-21T04:09:06.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:09:05 smithi195 ceph-mon[25840]: Saving service nfs.foo spec with placement count:2 2024-09-21T04:09:06.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:09:05 smithi195 ceph-mon[25840]: Creating key for client.nfs.foo.0.0.smithi089.dcjmqc 2024-09-21T04:09:06.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:09:05 smithi195 ceph-mon[25840]: Ensuring nfs.foo.0 is in the ganesha grace table 2024-09-21T04:09:06.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:09:05 smithi195 ceph-mon[25840]: Rados config object exists: conf-nfs.foo 2024-09-21T04:09:06.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:09:05 smithi195 ceph-mon[25840]: Creating key for client.nfs.foo.0.0.smithi089.dcjmqc-rgw 2024-09-21T04:09:06.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:09:05 smithi195 ceph-mon[25840]: Deploying daemon nfs.foo.0.0.smithi089.dcjmqc on smithi089 2024-09-21T04:09:06.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:09:05 smithi195 ceph-mon[25840]: osdmap e68: 8 total, 8 up, 8 in 2024-09-21T04:09:07.264 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:09:06 smithi089 ceph-mon[23733]: pgmap v231: 97 pgs: 1 creating+peering, 22 unknown, 74 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2024-09-21T04:09:07.264 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:09:06 smithi089 ceph-mon[23733]: mgrmap e20: smithi089.srlyxf(active, since 5m), standbys: smithi195.cgotzm 2024-09-21T04:09:07.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:09:06 smithi195 ceph-mon[25840]: pgmap v231: 97 pgs: 1 creating+peering, 22 unknown, 74 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2024-09-21T04:09:07.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:09:06 smithi195 ceph-mon[25840]: mgrmap e20: smithi089.srlyxf(active, since 5m), standbys: smithi195.cgotzm 2024-09-21T04:09:08.346 INFO:teuthology.orchestra.run.smithi089.stderr:Invalid command: Unexpected argument '--clusterid' 2024-09-21T04:09:08.346 INFO:teuthology.orchestra.run.smithi089.stderr:nfs export create cephfs [] [--readonly] [--client_addr ...] [--squash ] : Create a CephFS export 2024-09-21T04:09:08.346 INFO:teuthology.orchestra.run.smithi089.stderr:Error EINVAL: invalid command 2024-09-21T04:09:08.732 INFO:teuthology.orchestra.run.smithi089.stdout:{ 2024-09-21T04:09:08.732 INFO:teuthology.orchestra.run.smithi089.stdout: "bind": "/fake", 2024-09-21T04:09:08.732 INFO:teuthology.orchestra.run.smithi089.stdout: "fs": "foofs", 2024-09-21T04:09:08.732 INFO:teuthology.orchestra.run.smithi089.stdout: "path": "/", 2024-09-21T04:09:08.732 INFO:teuthology.orchestra.run.smithi089.stdout: "cluster": "foo", 2024-09-21T04:09:08.732 INFO:teuthology.orchestra.run.smithi089.stdout: "mode": "RW" 2024-09-21T04:09:08.732 INFO:teuthology.orchestra.run.smithi089.stdout:} 2024-09-21T04:09:09.264 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:09:08 smithi089 ceph-mon[23733]: pgmap v233: 97 pgs: 1 creating+peering, 2 unknown, 94 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 341 B/s wr, 0 op/s 2024-09-21T04:09:09.264 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:09:08 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' 2024-09-21T04:09:09.265 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:09:08 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi195.vhoayp", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]: dispatch 2024-09-21T04:09:09.265 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:09:08 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi195.vhoayp", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]': finished 2024-09-21T04:09:09.265 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:09:08 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]: dispatch 2024-09-21T04:09:09.265 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:09:08 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-09-21T04:09:09.265 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:09:08 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T04:09:09.265 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:09:08 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo, allow rw tag cephfs data=foofs", "mds", "allow rw path=/"], "format": "json"}]: dispatch 2024-09-21T04:09:09.265 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:09:08 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo, allow rw tag cephfs data=foofs", "mds", "allow rw path=/"], "format": "json"}]': finished 2024-09-21T04:09:09.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:09:08 smithi195 ceph-mon[25840]: pgmap v233: 97 pgs: 1 creating+peering, 2 unknown, 94 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 341 B/s wr, 0 op/s 2024-09-21T04:09:09.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:09:08 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' 2024-09-21T04:09:09.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:09:08 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi195.vhoayp", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]: dispatch 2024-09-21T04:09:09.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:09:08 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi195.vhoayp", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]': finished 2024-09-21T04:09:09.360 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:09:08 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]: dispatch 2024-09-21T04:09:09.360 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:09:08 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-09-21T04:09:09.360 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:09:08 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T04:09:09.360 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:09:08 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo, allow rw tag cephfs data=foofs", "mds", "allow rw path=/"], "format": "json"}]: dispatch 2024-09-21T04:09:09.360 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:09:08 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo, allow rw tag cephfs data=foofs", "mds", "allow rw path=/"], "format": "json"}]': finished 2024-09-21T04:09:09.411 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 1b9b79fc-77ce-11ef-baf5-efdc52797490 -- bash -c 'while ! ceph orch ls | grep nfs | grep 2/2 ; do sleep 1 ; done' 2024-09-21T04:09:10.015 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:09:09 smithi089 ceph-mon[23733]: Creating key for client.nfs.foo.1.0.smithi195.vhoayp 2024-09-21T04:09:10.015 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:09:09 smithi089 ceph-mon[23733]: Ensuring nfs.foo.1 is in the ganesha grace table 2024-09-21T04:09:10.015 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:09:09 smithi089 ceph-mon[23733]: from='client.14544 -' entity='client.admin' cmd=[{"prefix": "nfs export create cephfs", "fsname": "foofs", "cluster_id": "foo", "pseudo_path": "/fake", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T04:09:10.358 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:09:09 smithi195 ceph-mon[25840]: Creating key for client.nfs.foo.1.0.smithi195.vhoayp 2024-09-21T04:09:10.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:09:09 smithi195 ceph-mon[25840]: Ensuring nfs.foo.1 is in the ganesha grace table 2024-09-21T04:09:10.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:09:09 smithi195 ceph-mon[25840]: from='client.14544 -' entity='client.admin' cmd=[{"prefix": "nfs export create cephfs", "fsname": "foofs", "cluster_id": "foo", "pseudo_path": "/fake", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T04:09:11.014 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:09:10 smithi089 ceph-mon[23733]: pgmap v234: 97 pgs: 97 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 341 B/s wr, 0 op/s 2024-09-21T04:09:11.358 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:09:10 smithi195 ceph-mon[25840]: pgmap v234: 97 pgs: 97 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 341 B/s wr, 0 op/s 2024-09-21T04:09:11.946 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:09:11 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]: dispatch 2024-09-21T04:09:11.946 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:09:11 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-09-21T04:09:11.946 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:09:11 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi195.vhoayp-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-09-21T04:09:11.946 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:09:11 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi195.vhoayp-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-09-21T04:09:11.946 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:09:11 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T04:09:12.264 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:09:11 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]: dispatch 2024-09-21T04:09:12.265 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:09:11 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-09-21T04:09:12.265 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:09:11 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi195.vhoayp-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-09-21T04:09:12.265 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:09:11 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi195.vhoayp-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-09-21T04:09:12.265 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:09:11 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T04:09:13.264 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:09:12 smithi089 ceph-mon[23733]: pgmap v235: 97 pgs: 97 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 128 B/s rd, 256 B/s wr, 0 op/s 2024-09-21T04:09:13.265 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:09:12 smithi089 ceph-mon[23733]: from='client.14560 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T04:09:13.265 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:09:12 smithi089 ceph-mon[23733]: Rados config object exists: conf-nfs.foo 2024-09-21T04:09:13.265 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:09:12 smithi089 ceph-mon[23733]: Creating key for client.nfs.foo.1.0.smithi195.vhoayp-rgw 2024-09-21T04:09:13.265 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:09:12 smithi089 ceph-mon[23733]: Deploying daemon nfs.foo.1.0.smithi195.vhoayp on smithi195 2024-09-21T04:09:13.320 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:09:12 smithi195 ceph-mon[25840]: pgmap v235: 97 pgs: 97 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 128 B/s rd, 256 B/s wr, 0 op/s 2024-09-21T04:09:13.320 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:09:12 smithi195 ceph-mon[25840]: from='client.14560 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T04:09:13.320 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:09:12 smithi195 ceph-mon[25840]: Rados config object exists: conf-nfs.foo 2024-09-21T04:09:13.320 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:09:12 smithi195 ceph-mon[25840]: Creating key for client.nfs.foo.1.0.smithi195.vhoayp-rgw 2024-09-21T04:09:13.320 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:09:12 smithi195 ceph-mon[25840]: Deploying daemon nfs.foo.1.0.smithi195.vhoayp on smithi195 2024-09-21T04:09:13.961 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:09:13 smithi195 ceph-mon[25840]: from='client.14574 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T04:09:13.962 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:09:13 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.b", "id": [7, 2]}]: dispatch 2024-09-21T04:09:13.962 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:09:13 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.e", "id": [1, 2]}]: dispatch 2024-09-21T04:09:13.962 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:09:13 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.13", "id": [1, 5]}]: dispatch 2024-09-21T04:09:13.962 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:09:13 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [1, 2]}]: dispatch 2024-09-21T04:09:13.962 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:09:13 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi089.srlyxf/mirror_snapshot_schedule"}]: dispatch 2024-09-21T04:09:13.962 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:09:13 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi089.srlyxf/trash_purge_schedule"}]: dispatch 2024-09-21T04:09:14.264 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:09:13 smithi089 ceph-mon[23733]: from='client.14574 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T04:09:14.264 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:09:13 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.b", "id": [7, 2]}]: dispatch 2024-09-21T04:09:14.264 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:09:13 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.e", "id": [1, 2]}]: dispatch 2024-09-21T04:09:14.265 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:09:13 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.13", "id": [1, 5]}]: dispatch 2024-09-21T04:09:14.265 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:09:13 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [1, 2]}]: dispatch 2024-09-21T04:09:14.265 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:09:13 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi089.srlyxf/mirror_snapshot_schedule"}]: dispatch 2024-09-21T04:09:14.265 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:09:13 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi089.srlyxf/trash_purge_schedule"}]: dispatch 2024-09-21T04:09:15.264 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:09:14 smithi089 ceph-mon[23733]: pgmap v236: 97 pgs: 97 active+clean; 580 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.7 KiB/s rd, 1.7 KiB/s wr, 3 op/s 2024-09-21T04:09:15.265 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:09:14 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.b", "id": [7, 2]}]': finished 2024-09-21T04:09:15.265 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:09:14 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.e", "id": [1, 2]}]': finished 2024-09-21T04:09:15.265 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:09:14 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.13", "id": [1, 5]}]': finished 2024-09-21T04:09:15.265 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:09:14 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [1, 2]}]': finished 2024-09-21T04:09:15.265 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:09:14 smithi089 ceph-mon[23733]: osdmap e69: 8 total, 8 up, 8 in 2024-09-21T04:09:15.265 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:09:14 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' 2024-09-21T04:09:15.265 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:09:14 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T04:09:15.265 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:09:14 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T04:09:15.265 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:09:14 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T04:09:15.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:09:14 smithi195 ceph-mon[25840]: pgmap v236: 97 pgs: 97 active+clean; 580 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.7 KiB/s rd, 1.7 KiB/s wr, 3 op/s 2024-09-21T04:09:15.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:09:14 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.b", "id": [7, 2]}]': finished 2024-09-21T04:09:15.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:09:14 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.e", "id": [1, 2]}]': finished 2024-09-21T04:09:15.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:09:14 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.13", "id": [1, 5]}]': finished 2024-09-21T04:09:15.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:09:14 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [1, 2]}]': finished 2024-09-21T04:09:15.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:09:14 smithi195 ceph-mon[25840]: osdmap e69: 8 total, 8 up, 8 in 2024-09-21T04:09:15.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:09:14 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' 2024-09-21T04:09:15.360 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:09:14 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T04:09:15.360 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:09:14 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T04:09:15.360 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:09:14 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T04:09:16.235 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:09:15 smithi195 ceph-mon[25840]: from='client.14586 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T04:09:16.236 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:09:15 smithi195 ceph-mon[25840]: osdmap e70: 8 total, 8 up, 8 in 2024-09-21T04:09:16.259 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:09:15 smithi089 ceph-mon[23733]: from='client.14586 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T04:09:16.259 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:09:15 smithi089 ceph-mon[23733]: osdmap e70: 8 total, 8 up, 8 in 2024-09-21T04:09:17.256 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:09:16 smithi089 ceph-mon[23733]: pgmap v239: 97 pgs: 2 peering, 95 active+clean; 580 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.7 KiB/s rd, 1.6 KiB/s wr, 3 op/s 2024-09-21T04:09:17.256 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:09:16 smithi089 ceph-mon[23733]: from='client.14594 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T04:09:17.257 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:09:16 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' 2024-09-21T04:09:17.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:09:16 smithi195 ceph-mon[25840]: pgmap v239: 97 pgs: 2 peering, 95 active+clean; 580 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.7 KiB/s rd, 1.6 KiB/s wr, 3 op/s 2024-09-21T04:09:17.360 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:09:16 smithi195 ceph-mon[25840]: from='client.14594 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T04:09:17.360 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:09:16 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' 2024-09-21T04:09:18.265 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:09:17 smithi089 ceph-mon[23733]: from='client.14598 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T04:09:18.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:09:17 smithi195 ceph-mon[25840]: from='client.14598 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T04:09:19.264 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:09:18 smithi089 ceph-mon[23733]: pgmap v240: 97 pgs: 2 peering, 95 active+clean; 581 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 5.1 KiB/s rd, 3.1 KiB/s wr, 8 op/s; 1 B/s, 0 keys/s, 0 objects/s recovering 2024-09-21T04:09:19.264 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:09:18 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' 2024-09-21T04:09:19.358 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:09:18 smithi195 ceph-mon[25840]: pgmap v240: 97 pgs: 2 peering, 95 active+clean; 581 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 5.1 KiB/s rd, 3.1 KiB/s wr, 8 op/s; 1 B/s, 0 keys/s, 0 objects/s recovering 2024-09-21T04:09:19.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:09:18 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' 2024-09-21T04:09:19.457 INFO:teuthology.orchestra.run.smithi089.stdout:nfs.foo ?:2049 2/2 2s ago 15s count:2 2024-09-21T04:09:20.170 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:09:19 smithi089 ceph-mon[23733]: from='client.14602 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T04:09:20.170 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:09:19 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' 2024-09-21T04:09:20.170 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:09:19 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' 2024-09-21T04:09:20.170 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:09:19 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' 2024-09-21T04:09:20.170 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:09:19 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T04:09:20.170 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:09:19 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T04:09:20.170 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:09:19 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T04:09:20.170 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:09:19 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' 2024-09-21T04:09:20.170 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:09:19 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' 2024-09-21T04:09:20.193 INFO:teuthology.run_tasks:Running task vip.exec... 2024-09-21T04:09:20.208 INFO:tasks.vip:Running commands on role host.a host ubuntu@smithi089.front.sepia.ceph.com 2024-09-21T04:09:20.208 DEBUG:teuthology.orchestra.run.smithi089:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c 'mkdir /mnt/foo' 2024-09-21T04:09:20.234 INFO:teuthology.orchestra.run.smithi089.stderr:+ mkdir /mnt/foo 2024-09-21T04:09:20.241 DEBUG:teuthology.orchestra.run.smithi089:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c 'while ! mount -t nfs $(hostname):/fake /mnt/foo -o sync ; do sleep 5 ; done' 2024-09-21T04:09:20.305 INFO:teuthology.orchestra.run.smithi089.stderr:++ hostname 2024-09-21T04:09:20.305 INFO:teuthology.orchestra.run.smithi089.stderr:+ mount -t nfs smithi089:/fake /mnt/foo -o sync 2024-09-21T04:09:20.358 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:09:19 smithi195 ceph-mon[25840]: from='client.14602 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T04:09:20.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:09:19 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' 2024-09-21T04:09:20.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:09:19 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' 2024-09-21T04:09:20.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:09:19 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' 2024-09-21T04:09:20.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:09:19 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T04:09:20.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:09:19 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T04:09:20.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:09:19 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T04:09:20.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:09:19 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' 2024-09-21T04:09:20.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:09:19 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' 2024-09-21T04:09:20.730 DEBUG:teuthology.orchestra.run.smithi089:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c 'echo test > /mnt/foo/testfile' 2024-09-21T04:09:20.795 INFO:teuthology.orchestra.run.smithi089.stderr:+ echo test 2024-09-21T04:09:20.805 DEBUG:teuthology.orchestra.run.smithi089:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c sync 2024-09-21T04:09:20.870 INFO:teuthology.orchestra.run.smithi089.stderr:+ sync 2024-09-21T04:09:21.084 INFO:teuthology.run_tasks:Running task parallel... 2024-09-21T04:09:21.097 INFO:teuthology.task.parallel:starting parallel... 2024-09-21T04:09:21.097 INFO:teuthology.task.parallel:In parallel, running task sequential... 2024-09-21T04:09:21.098 INFO:teuthology.task.sequential:In sequential, running task cephadm.shell... 2024-09-21T04:09:21.098 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi089.front.sepia.ceph.com 2024-09-21T04:09:21.098 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 1b9b79fc-77ce-11ef-baf5-efdc52797490 -e sha1=f36283e7c91cac1c01cb9a7c4ff21bd697dbd4f2 -- bash -c 'ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force' 2024-09-21T04:09:21.099 INFO:teuthology.task.parallel:In parallel, running task sequential... 2024-09-21T04:09:21.099 INFO:teuthology.task.sequential:In sequential, running task exec... 2024-09-21T04:09:21.100 INFO:teuthology.task.exec:Executing custom commands... 2024-09-21T04:09:21.100 INFO:teuthology.task.exec:Running commands on role host.a host ubuntu@smithi089.front.sepia.ceph.com 2024-09-21T04:09:21.100 DEBUG:teuthology.orchestra.run.smithi089:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'cd /mnt/foo && dbench 5 -t 600 || true' 2024-09-21T04:09:21.146 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:09:20 smithi089 ceph-mon[23733]: pgmap v241: 97 pgs: 2 peering, 95 active+clean; 581 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 5.1 KiB/s rd, 3.1 KiB/s wr, 8 op/s; 1 B/s, 0 keys/s, 0 objects/s recovering 2024-09-21T04:09:21.146 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:09:20 smithi089 ceph-mon[23733]: from='client.14606 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T04:09:21.172 INFO:teuthology.orchestra.run.smithi089.stdout:dbench version 4.00 - Copyright Andrew Tridgell 1999-2004 2024-09-21T04:09:21.172 INFO:teuthology.orchestra.run.smithi089.stdout: 2024-09-21T04:09:21.172 INFO:teuthology.orchestra.run.smithi089.stdout:Running for 600 seconds with load '/usr/share/dbench/client.txt' and minimum warmup 120 secs 2024-09-21T04:09:21.173 INFO:teuthology.orchestra.run.smithi089.stdout:4 of 5 processes prepared for launch 0 sec 2024-09-21T04:09:21.273 INFO:teuthology.orchestra.run.smithi089.stdout:5 of 5 processes prepared for launch 0 sec 2024-09-21T04:09:21.274 INFO:teuthology.orchestra.run.smithi089.stdout:releasing clients 2024-09-21T04:09:21.358 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:09:20 smithi195 ceph-mon[25840]: pgmap v241: 97 pgs: 2 peering, 95 active+clean; 581 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 5.1 KiB/s rd, 3.1 KiB/s wr, 8 op/s; 1 B/s, 0 keys/s, 0 objects/s recovering 2024-09-21T04:09:21.358 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:09:20 smithi195 ceph-mon[25840]: from='client.14606 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T04:09:22.274 INFO:teuthology.orchestra.run.smithi089.stdout: 5 169 34.54 MB/sec warmup 1 sec latency 31.835 ms 2024-09-21T04:09:23.215 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:09:22 smithi089 ceph-mon[23733]: pgmap v242: 97 pgs: 97 active+clean; 581 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 3.4 KiB/s rd, 1.5 KiB/s wr, 5 op/s; 1 B/s, 0 keys/s, 0 objects/s recovering 2024-09-21T04:09:23.273 INFO:teuthology.orchestra.run.smithi089.stdout: 5 303 27.87 MB/sec warmup 2 sec latency 39.286 ms 2024-09-21T04:09:23.358 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:09:22 smithi195 ceph-mon[25840]: pgmap v242: 97 pgs: 97 active+clean; 581 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 3.4 KiB/s rd, 1.5 KiB/s wr, 5 op/s; 1 B/s, 0 keys/s, 0 objects/s recovering 2024-09-21T04:09:24.273 INFO:teuthology.orchestra.run.smithi089.stdout: 5 486 28.22 MB/sec warmup 3 sec latency 31.428 ms 2024-09-21T04:09:24.699 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 1b9b79fc-77ce-11ef-baf5-efdc52797490 -e sha1=f36283e7c91cac1c01cb9a7c4ff21bd697dbd4f2 -- bash -c 'ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force' 2024-09-21T04:09:25.024 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:09:24 smithi089 ceph-mon[23733]: pgmap v243: 97 pgs: 97 active+clean; 17 MiB data, 67 MiB used, 715 GiB / 715 GiB avail; 2.9 KiB/s rd, 1.9 MiB/s wr, 88 op/s; 1 B/s, 0 keys/s, 0 objects/s recovering 2024-09-21T04:09:25.273 INFO:teuthology.orchestra.run.smithi089.stdout: 5 701 26.45 MB/sec warmup 4 sec latency 31.362 ms 2024-09-21T04:09:25.358 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:09:24 smithi195 ceph-mon[25840]: pgmap v243: 97 pgs: 97 active+clean; 17 MiB data, 67 MiB used, 715 GiB / 715 GiB avail; 2.9 KiB/s rd, 1.9 MiB/s wr, 88 op/s; 1 B/s, 0 keys/s, 0 objects/s recovering 2024-09-21T04:09:26.274 INFO:teuthology.orchestra.run.smithi089.stdout: 5 1083 22.68 MB/sec warmup 5 sec latency 39.308 ms 2024-09-21T04:09:27.036 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:09:26 smithi089 ceph-mon[23733]: pgmap v244: 97 pgs: 97 active+clean; 27 MiB data, 72 MiB used, 715 GiB / 715 GiB avail; 2.7 KiB/s rd, 2.7 MiB/s wr, 98 op/s; 1 B/s, 0 keys/s, 0 objects/s recovering 2024-09-21T04:09:27.273 INFO:teuthology.orchestra.run.smithi089.stdout: 5 1845 21.97 MB/sec warmup 6 sec latency 40.159 ms 2024-09-21T04:09:27.358 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:09:26 smithi195 ceph-mon[25840]: pgmap v244: 97 pgs: 97 active+clean; 27 MiB data, 72 MiB used, 715 GiB / 715 GiB avail; 2.7 KiB/s rd, 2.7 MiB/s wr, 98 op/s; 1 B/s, 0 keys/s, 0 objects/s recovering 2024-09-21T04:09:28.274 INFO:teuthology.orchestra.run.smithi089.stdout: 5 2339 19.78 MB/sec warmup 7 sec latency 34.456 ms 2024-09-21T04:09:29.037 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:09:29 smithi089 ceph-mon[23733]: pgmap v245: 97 pgs: 97 active+clean; 104 MiB data, 178 MiB used, 715 GiB / 715 GiB avail; 2.4 KiB/s rd, 9.2 MiB/s wr, 428 op/s; 1 B/s, 0 keys/s, 0 objects/s recovering 2024-09-21T04:09:29.274 INFO:teuthology.orchestra.run.smithi089.stdout: 5 2696 18.97 MB/sec warmup 8 sec latency 45.321 ms 2024-09-21T04:09:29.358 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:09:29 smithi195 ceph-mon[25840]: pgmap v245: 97 pgs: 97 active+clean; 104 MiB data, 178 MiB used, 715 GiB / 715 GiB avail; 2.4 KiB/s rd, 9.2 MiB/s wr, 428 op/s; 1 B/s, 0 keys/s, 0 objects/s recovering 2024-09-21T04:09:29.409 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 1b9b79fc-77ce-11ef-baf5-efdc52797490 -e sha1=f36283e7c91cac1c01cb9a7c4ff21bd697dbd4f2 -- bash -c 'ceph config set global log_to_journald false --force' 2024-09-21T04:09:30.274 INFO:teuthology.orchestra.run.smithi089.stdout: 5 3208 19.21 MB/sec warmup 9 sec latency 53.395 ms 2024-09-21T04:09:31.075 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:09:31 smithi089 ceph-mon[23733]: pgmap v246: 97 pgs: 97 active+clean; 114 MiB data, 211 MiB used, 715 GiB / 715 GiB avail; 938 B/s rd, 10 MiB/s wr, 519 op/s; 0 B/s, 0 objects/s recovering 2024-09-21T04:09:31.274 INFO:teuthology.orchestra.run.smithi089.stdout: 5 3691 19.52 MB/sec warmup 10 sec latency 73.541 ms 2024-09-21T04:09:31.358 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:09:31 smithi195 ceph-mon[25840]: pgmap v246: 97 pgs: 97 active+clean; 114 MiB data, 211 MiB used, 715 GiB / 715 GiB avail; 938 B/s rd, 10 MiB/s wr, 519 op/s; 0 B/s, 0 objects/s recovering 2024-09-21T04:09:32.274 INFO:teuthology.orchestra.run.smithi089.stdout: 5 3852 18.11 MB/sec warmup 11 sec latency 244.756 ms 2024-09-21T04:09:33.136 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:09:33 smithi089 ceph-mon[23733]: pgmap v247: 97 pgs: 97 active+clean; 123 MiB data, 243 MiB used, 715 GiB / 715 GiB avail; 938 B/s rd, 12 MiB/s wr, 626 op/s 2024-09-21T04:09:33.274 INFO:teuthology.orchestra.run.smithi089.stdout: 5 4376 16.91 MB/sec warmup 12 sec latency 42.318 ms 2024-09-21T04:09:33.358 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:09:33 smithi195 ceph-mon[25840]: pgmap v247: 97 pgs: 97 active+clean; 123 MiB data, 243 MiB used, 715 GiB / 715 GiB avail; 938 B/s rd, 12 MiB/s wr, 626 op/s 2024-09-21T04:09:34.274 INFO:teuthology.orchestra.run.smithi089.stdout: 5 5099 17.39 MB/sec warmup 13 sec latency 29.165 ms 2024-09-21T04:09:34.946 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 1b9b79fc-77ce-11ef-baf5-efdc52797490 -e sha1=f36283e7c91cac1c01cb9a7c4ff21bd697dbd4f2 -- bash -c 'ceph mgr module enable nfs --force' 2024-09-21T04:09:35.167 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:09:35 smithi089 ceph-mon[23733]: pgmap v248: 97 pgs: 97 active+clean; 127 MiB data, 207 MiB used, 715 GiB / 715 GiB avail; 938 B/s rd, 13 MiB/s wr, 665 op/s 2024-09-21T04:09:35.274 INFO:teuthology.orchestra.run.smithi089.stdout: 5 5858 16.62 MB/sec warmup 14 sec latency 37.756 ms 2024-09-21T04:09:35.358 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:09:35 smithi195 ceph-mon[25840]: pgmap v248: 97 pgs: 97 active+clean; 127 MiB data, 207 MiB used, 715 GiB / 715 GiB avail; 938 B/s rd, 13 MiB/s wr, 665 op/s 2024-09-21T04:09:36.274 INFO:teuthology.orchestra.run.smithi089.stdout: 5 6228 16.44 MB/sec warmup 15 sec latency 60.008 ms 2024-09-21T04:09:37.219 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:09:37 smithi089 ceph-mon[23733]: pgmap v249: 97 pgs: 97 active+clean; 128 MiB data, 229 MiB used, 715 GiB / 715 GiB avail; 3.5 KiB/s rd, 11 MiB/s wr, 633 op/s 2024-09-21T04:09:37.274 INFO:teuthology.orchestra.run.smithi089.stdout: 5 6767 16.77 MB/sec warmup 16 sec latency 43.280 ms 2024-09-21T04:09:37.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:09:37 smithi195 ceph-mon[25840]: pgmap v249: 97 pgs: 97 active+clean; 128 MiB data, 229 MiB used, 715 GiB / 715 GiB avail; 3.5 KiB/s rd, 11 MiB/s wr, 633 op/s 2024-09-21T04:09:38.274 INFO:teuthology.orchestra.run.smithi089.stdout: 5 7218 17.11 MB/sec warmup 17 sec latency 41.174 ms 2024-09-21T04:09:39.032 INFO:teuthology.orchestra.run.smithi089.stderr:module 'nfs' is already enabled 2024-09-21T04:09:39.247 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:09:39 smithi089 ceph-mon[23733]: pgmap v250: 97 pgs: 97 active+clean; 134 MiB data, 297 MiB used, 715 GiB / 715 GiB avail; 3.7 KiB/s rd, 13 MiB/s wr, 906 op/s 2024-09-21T04:09:39.247 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:09:39 smithi089 ceph-mon[23733]: from='client.? 172.21.15.89:0/1768705443' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "nfs", "force": true}]: dispatch 2024-09-21T04:09:39.274 INFO:teuthology.orchestra.run.smithi089.stdout: 5 7461 16.41 MB/sec warmup 18 sec latency 51.874 ms 2024-09-21T04:09:39.358 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:09:39 smithi195 ceph-mon[25840]: pgmap v250: 97 pgs: 97 active+clean; 134 MiB data, 297 MiB used, 715 GiB / 715 GiB avail; 3.7 KiB/s rd, 13 MiB/s wr, 906 op/s 2024-09-21T04:09:39.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:09:39 smithi195 ceph-mon[25840]: from='client.? 172.21.15.89:0/1768705443' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "nfs", "force": true}]: dispatch 2024-09-21T04:09:40.025 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 1b9b79fc-77ce-11ef-baf5-efdc52797490 -e sha1=f36283e7c91cac1c01cb9a7c4ff21bd697dbd4f2 -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1' 2024-09-21T04:09:40.265 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:09:40 smithi089 ceph-mon[23733]: from='client.? 172.21.15.89:0/1768705443' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "nfs", "force": true}]': finished 2024-09-21T04:09:40.266 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:09:40 smithi089 ceph-mon[23733]: mgrmap e21: smithi089.srlyxf(active, since 6m), standbys: smithi195.cgotzm 2024-09-21T04:09:40.274 INFO:teuthology.orchestra.run.smithi089.stdout: 5 7805 15.65 MB/sec warmup 19 sec latency 65.442 ms 2024-09-21T04:09:40.358 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:09:40 smithi195 ceph-mon[25840]: from='client.? 172.21.15.89:0/1768705443' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "nfs", "force": true}]': finished 2024-09-21T04:09:40.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:09:40 smithi195 ceph-mon[25840]: mgrmap e21: smithi089.srlyxf(active, since 6m), standbys: smithi195.cgotzm 2024-09-21T04:09:41.038 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:09:41 smithi089 ceph-mon[23733]: pgmap v251: 97 pgs: 97 active+clean; 134 MiB data, 317 MiB used, 715 GiB / 715 GiB avail; 8.4 KiB/s rd, 6.5 MiB/s wr, 579 op/s 2024-09-21T04:09:41.274 INFO:teuthology.orchestra.run.smithi089.stdout: 5 8120 15.13 MB/sec warmup 20 sec latency 54.282 ms 2024-09-21T04:09:41.358 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:09:41 smithi195 ceph-mon[25840]: pgmap v251: 97 pgs: 97 active+clean; 134 MiB data, 317 MiB used, 715 GiB / 715 GiB avail; 8.4 KiB/s rd, 6.5 MiB/s wr, 579 op/s 2024-09-21T04:09:42.274 INFO:teuthology.orchestra.run.smithi089.stdout: 5 8450 14.88 MB/sec warmup 21 sec latency 63.451 ms 2024-09-21T04:09:42.938 INFO:teuthology.orchestra.run.smithi089.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:f36283e7c91cac1c01cb9a7c4ff21bd697dbd4f2 2024-09-21T04:09:43.089 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:09:43 smithi089 ceph-mon[23733]: pgmap v252: 97 pgs: 97 active+clean; 145 MiB data, 355 MiB used, 715 GiB / 715 GiB avail; 7.7 KiB/s rd, 7.4 MiB/s wr, 666 op/s 2024-09-21T04:09:43.089 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:09:43 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' 2024-09-21T04:09:43.090 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:09:43 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T04:09:43.090 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:09:43 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T04:09:43.090 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:09:43 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T04:09:43.274 INFO:teuthology.orchestra.run.smithi089.stdout: 5 8871 14.67 MB/sec warmup 22 sec latency 67.910 ms 2024-09-21T04:09:43.302 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:09:43 smithi195 ceph-mon[25840]: pgmap v252: 97 pgs: 97 active+clean; 145 MiB data, 355 MiB used, 715 GiB / 715 GiB avail; 7.7 KiB/s rd, 7.4 MiB/s wr, 666 op/s 2024-09-21T04:09:43.302 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:09:43 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' 2024-09-21T04:09:43.302 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:09:43 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T04:09:43.302 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:09:43 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T04:09:43.302 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:09:43 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T04:09:44.044 INFO:teuthology.task.sequential:In sequential, running task cephadm.shell... 2024-09-21T04:09:44.044 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi089.front.sepia.ceph.com 2024-09-21T04:09:44.044 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 1b9b79fc-77ce-11ef-baf5-efdc52797490 -e sha1=f36283e7c91cac1c01cb9a7c4ff21bd697dbd4f2 -- bash -c 'while ceph orch upgrade status | jq '"'"'.in_progress'"'"' | grep true && ! ceph orch upgrade status | jq '"'"'.message'"'"' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; ceph health detail ; sleep 30 ; done' 2024-09-21T04:09:44.093 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:09:44 smithi089 ceph-mon[23733]: from='client.14626 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:f36283e7c91cac1c01cb9a7c4ff21bd697dbd4f2", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T04:09:44.093 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:09:44 smithi089 ceph-mon[23733]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:f36283e7c91cac1c01cb9a7c4ff21bd697dbd4f2 2024-09-21T04:09:44.093 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:09:44 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' 2024-09-21T04:09:44.093 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:09:44 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' 2024-09-21T04:09:44.275 INFO:teuthology.orchestra.run.smithi089.stdout: 5 9356 14.23 MB/sec warmup 23 sec latency 99.444 ms 2024-09-21T04:09:44.358 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:09:44 smithi195 ceph-mon[25840]: from='client.14626 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:f36283e7c91cac1c01cb9a7c4ff21bd697dbd4f2", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T04:09:44.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:09:44 smithi195 ceph-mon[25840]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:f36283e7c91cac1c01cb9a7c4ff21bd697dbd4f2 2024-09-21T04:09:44.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:09:44 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' 2024-09-21T04:09:44.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:09:44 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' 2024-09-21T04:09:45.113 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:09:45 smithi089 ceph-mon[23733]: pgmap v253: 97 pgs: 97 active+clean; 146 MiB data, 390 MiB used, 715 GiB / 715 GiB avail; 7.7 KiB/s rd, 6.5 MiB/s wr, 591 op/s 2024-09-21T04:09:45.113 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:09:45 smithi089 ceph-mon[23733]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:f36283e7c91cac1c01cb9a7c4ff21bd697dbd4f2 2024-09-21T04:09:45.275 INFO:teuthology.orchestra.run.smithi089.stdout: 5 9646 14.06 MB/sec warmup 24 sec latency 43.228 ms 2024-09-21T04:09:45.358 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:09:45 smithi195 ceph-mon[25840]: pgmap v253: 97 pgs: 97 active+clean; 146 MiB data, 390 MiB used, 715 GiB / 715 GiB avail; 7.7 KiB/s rd, 6.5 MiB/s wr, 591 op/s 2024-09-21T04:09:45.358 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:09:45 smithi195 ceph-mon[25840]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:f36283e7c91cac1c01cb9a7c4ff21bd697dbd4f2 2024-09-21T04:09:46.285 INFO:teuthology.orchestra.run.smithi089.stdout: 5 10278 14.50 MB/sec warmup 25 sec latency 42.432 ms 2024-09-21T04:09:47.121 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:09:47 smithi089 ceph-mon[23733]: pgmap v254: 97 pgs: 97 active+clean; 145 MiB data, 391 MiB used, 715 GiB / 715 GiB avail; 12 KiB/s rd, 5.7 MiB/s wr, 563 op/s 2024-09-21T04:09:47.285 INFO:teuthology.orchestra.run.smithi089.stdout: 5 10746 14.71 MB/sec warmup 26 sec latency 43.150 ms 2024-09-21T04:09:47.358 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:09:47 smithi195 ceph-mon[25840]: pgmap v254: 97 pgs: 97 active+clean; 145 MiB data, 391 MiB used, 715 GiB / 715 GiB avail; 12 KiB/s rd, 5.7 MiB/s wr, 563 op/s 2024-09-21T04:09:47.441 INFO:teuthology.orchestra.run.smithi089.stdout:true 2024-09-21T04:09:48.285 INFO:teuthology.orchestra.run.smithi089.stdout: 5 11006 14.47 MB/sec warmup 27 sec latency 37.463 ms 2024-09-21T04:09:48.473 INFO:teuthology.orchestra.run.smithi089.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-21T04:09:48.473 INFO:teuthology.orchestra.run.smithi089.stdout:alertmanager.smithi089 smithi089 *:9093,9094 running (5m) 29s ago 6m 22.6M - ba2b418f427c 703c928c9523 2024-09-21T04:09:48.473 INFO:teuthology.orchestra.run.smithi089.stdout:crash.smithi089 smithi089 running (6m) 29s ago 6m 6409k - 17.2.0 e1d6a67b021e d1e0a8969f78 2024-09-21T04:09:48.474 INFO:teuthology.orchestra.run.smithi089.stdout:crash.smithi195 smithi195 running (6m) 31s ago 6m 6400k - 17.2.0 e1d6a67b021e 7e213ff76084 2024-09-21T04:09:48.474 INFO:teuthology.orchestra.run.smithi089.stdout:grafana.smithi089 smithi089 *:3000 running (5m) 29s ago 6m 53.2M - 8.3.5 dad864ee21e9 547de193a0ee 2024-09-21T04:09:48.474 INFO:teuthology.orchestra.run.smithi089.stdout:mds.foofs.smithi089.osufws smithi089 running (62s) 29s ago 62s 13.3M - 17.2.0 e1d6a67b021e 664d41859bea 2024-09-21T04:09:48.474 INFO:teuthology.orchestra.run.smithi089.stdout:mds.foofs.smithi195.hijxre smithi195 running (60s) 31s ago 59s 14.2M - 17.2.0 e1d6a67b021e 7bc0c6542d12 2024-09-21T04:09:48.474 INFO:teuthology.orchestra.run.smithi089.stdout:mgr.smithi089.srlyxf smithi089 *:9283 running (8m) 29s ago 8m 461M - 17.2.0 e1d6a67b021e 410bb00e916d 2024-09-21T04:09:48.474 INFO:teuthology.orchestra.run.smithi089.stdout:mgr.smithi195.cgotzm smithi195 *:8443,9283 running (6m) 31s ago 6m 412M - 17.2.0 e1d6a67b021e f2b03f4a2ef8 2024-09-21T04:09:48.474 INFO:teuthology.orchestra.run.smithi089.stdout:mon.smithi089 smithi089 running (8m) 29s ago 8m 46.7M 2048M 17.2.0 e1d6a67b021e 1d2b3719e46e 2024-09-21T04:09:48.474 INFO:teuthology.orchestra.run.smithi089.stdout:mon.smithi195 smithi195 running (6m) 31s ago 6m 37.7M 2048M 17.2.0 e1d6a67b021e 7f86c5b5d21e 2024-09-21T04:09:48.474 INFO:teuthology.orchestra.run.smithi089.stdout:nfs.foo.0.0.smithi089.dcjmqc smithi089 *:2049 running (40s) 29s ago 39s 46.8M - 4.0 e1d6a67b021e 98550f94e773 2024-09-21T04:09:48.474 INFO:teuthology.orchestra.run.smithi089.stdout:nfs.foo.1.0.smithi195.vhoayp smithi195 *:2049 running (34s) 31s ago 34s 16.0M - 4.0 e1d6a67b021e c60a4040130a 2024-09-21T04:09:48.474 INFO:teuthology.orchestra.run.smithi089.stdout:node-exporter.smithi089 smithi089 *:9100 running (6m) 29s ago 6m 17.5M - 1dbe0e931976 5b166976fc1c 2024-09-21T04:09:48.474 INFO:teuthology.orchestra.run.smithi089.stdout:node-exporter.smithi195 smithi195 *:9100 running (6m) 31s ago 6m 17.0M - 1dbe0e931976 3448624c239b 2024-09-21T04:09:48.474 INFO:teuthology.orchestra.run.smithi089.stdout:osd.0 smithi089 running (5m) 29s ago 5m 42.6M 3215M 17.2.0 e1d6a67b021e db7c0e5020e7 2024-09-21T04:09:48.475 INFO:teuthology.orchestra.run.smithi089.stdout:osd.1 smithi089 running (5m) 29s ago 5m 43.6M 3215M 17.2.0 e1d6a67b021e f9ec697d08d0 2024-09-21T04:09:48.475 INFO:teuthology.orchestra.run.smithi089.stdout:osd.2 smithi089 running (4m) 29s ago 4m 40.8M 3215M 17.2.0 e1d6a67b021e cfa000948859 2024-09-21T04:09:48.475 INFO:teuthology.orchestra.run.smithi089.stdout:osd.3 smithi089 running (4m) 29s ago 4m 40.6M 3215M 17.2.0 e1d6a67b021e 31c610ed5815 2024-09-21T04:09:48.475 INFO:teuthology.orchestra.run.smithi089.stdout:osd.4 smithi195 running (3m) 31s ago 3m 39.3M 3984M 17.2.0 e1d6a67b021e f00e18a1f220 2024-09-21T04:09:48.475 INFO:teuthology.orchestra.run.smithi089.stdout:osd.5 smithi195 running (3m) 31s ago 3m 40.0M 3984M 17.2.0 e1d6a67b021e cecfd6ff1eee 2024-09-21T04:09:48.475 INFO:teuthology.orchestra.run.smithi089.stdout:osd.6 smithi195 running (2m) 31s ago 2m 39.1M 3984M 17.2.0 e1d6a67b021e 0d7db362cc23 2024-09-21T04:09:48.475 INFO:teuthology.orchestra.run.smithi089.stdout:osd.7 smithi195 running (2m) 31s ago 2m 40.5M 3984M 17.2.0 e1d6a67b021e 0109b3fea188 2024-09-21T04:09:48.475 INFO:teuthology.orchestra.run.smithi089.stdout:prometheus.smithi089 smithi089 *:9095 running (5m) 29s ago 5m 62.8M - 514e6a882f6e 9ed6ce8f2de3 2024-09-21T04:09:49.097 INFO:teuthology.orchestra.run.smithi089.stdout:{ 2024-09-21T04:09:49.097 INFO:teuthology.orchestra.run.smithi089.stdout: "mon": { 2024-09-21T04:09:49.097 INFO:teuthology.orchestra.run.smithi089.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-21T04:09:49.097 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2024-09-21T04:09:49.097 INFO:teuthology.orchestra.run.smithi089.stdout: "mgr": { 2024-09-21T04:09:49.097 INFO:teuthology.orchestra.run.smithi089.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-21T04:09:49.097 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2024-09-21T04:09:49.097 INFO:teuthology.orchestra.run.smithi089.stdout: "osd": { 2024-09-21T04:09:49.097 INFO:teuthology.orchestra.run.smithi089.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-09-21T04:09:49.097 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2024-09-21T04:09:49.097 INFO:teuthology.orchestra.run.smithi089.stdout: "mds": { 2024-09-21T04:09:49.098 INFO:teuthology.orchestra.run.smithi089.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-21T04:09:49.098 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2024-09-21T04:09:49.098 INFO:teuthology.orchestra.run.smithi089.stdout: "overall": { 2024-09-21T04:09:49.098 INFO:teuthology.orchestra.run.smithi089.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 14 2024-09-21T04:09:49.098 INFO:teuthology.orchestra.run.smithi089.stdout: } 2024-09-21T04:09:49.098 INFO:teuthology.orchestra.run.smithi089.stdout:} 2024-09-21T04:09:49.134 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:09:49 smithi089 ceph-mon[23733]: pgmap v255: 97 pgs: 97 active+clean; 147 MiB data, 419 MiB used, 715 GiB / 715 GiB avail; 9.7 KiB/s rd, 7.5 MiB/s wr, 753 op/s 2024-09-21T04:09:49.134 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:09:49 smithi089 ceph-mon[23733]: from='client.14630 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T04:09:49.135 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:09:49 smithi089 ceph-mon[23733]: from='client.14634 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T04:09:49.285 INFO:teuthology.orchestra.run.smithi089.stdout: 5 11417 14.05 MB/sec warmup 28 sec latency 45.106 ms 2024-09-21T04:09:49.358 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:09:49 smithi195 ceph-mon[25840]: pgmap v255: 97 pgs: 97 active+clean; 147 MiB data, 419 MiB used, 715 GiB / 715 GiB avail; 9.7 KiB/s rd, 7.5 MiB/s wr, 753 op/s 2024-09-21T04:09:49.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:09:49 smithi195 ceph-mon[25840]: from='client.14630 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T04:09:49.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:09:49 smithi195 ceph-mon[25840]: from='client.14634 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T04:09:49.549 INFO:teuthology.orchestra.run.smithi089.stdout:{ 2024-09-21T04:09:49.549 INFO:teuthology.orchestra.run.smithi089.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph:f36283e7c91cac1c01cb9a7c4ff21bd697dbd4f2", 2024-09-21T04:09:49.549 INFO:teuthology.orchestra.run.smithi089.stdout: "in_progress": true, 2024-09-21T04:09:49.549 INFO:teuthology.orchestra.run.smithi089.stdout: "services_complete": [], 2024-09-21T04:09:49.549 INFO:teuthology.orchestra.run.smithi089.stdout: "progress": "", 2024-09-21T04:09:49.549 INFO:teuthology.orchestra.run.smithi089.stdout: "message": "Doing first pull of quay.ceph.io/ceph-ci/ceph:f36283e7c91cac1c01cb9a7c4ff21bd697dbd4f2 image" 2024-09-21T04:09:49.549 INFO:teuthology.orchestra.run.smithi089.stdout:} 2024-09-21T04:09:50.036 INFO:teuthology.orchestra.run.smithi089.stdout:HEALTH_OK 2024-09-21T04:09:50.073 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:09:50 smithi089 ceph-mon[23733]: from='client.14638 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T04:09:50.073 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:09:50 smithi089 ceph-mon[23733]: from='client.? 172.21.15.89:0/808793133' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T04:09:50.073 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:09:50 smithi089 ceph-mon[23733]: from='client.? 172.21.15.89:0/3577389272' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-21T04:09:50.285 INFO:teuthology.orchestra.run.smithi089.stdout: 5 11683 13.76 MB/sec warmup 29 sec latency 48.361 ms 2024-09-21T04:09:50.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:09:50 smithi195 ceph-mon[25840]: from='client.14638 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T04:09:50.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:09:50 smithi195 ceph-mon[25840]: from='client.? 172.21.15.89:0/808793133' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T04:09:50.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:09:50 smithi195 ceph-mon[25840]: from='client.? 172.21.15.89:0/3577389272' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-21T04:09:51.159 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:09:51 smithi089 ceph-mon[23733]: pgmap v256: 97 pgs: 97 active+clean; 151 MiB data, 419 MiB used, 715 GiB / 715 GiB avail; 12 KiB/s rd, 5.4 MiB/s wr, 492 op/s 2024-09-21T04:09:51.159 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:09:51 smithi089 ceph-mon[23733]: from='client.24387 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T04:09:51.285 INFO:teuthology.orchestra.run.smithi089.stdout: 5 12021 13.70 MB/sec warmup 30 sec latency 39.777 ms 2024-09-21T04:09:51.358 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:09:51 smithi195 ceph-mon[25840]: pgmap v256: 97 pgs: 97 active+clean; 151 MiB data, 419 MiB used, 715 GiB / 715 GiB avail; 12 KiB/s rd, 5.4 MiB/s wr, 492 op/s 2024-09-21T04:09:51.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:09:51 smithi195 ceph-mon[25840]: from='client.24387 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T04:09:52.285 INFO:teuthology.orchestra.run.smithi089.stdout: 5 12482 13.50 MB/sec warmup 31 sec latency 47.279 ms 2024-09-21T04:09:53.180 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:09:53 smithi089 ceph-mon[23733]: pgmap v257: 97 pgs: 97 active+clean; 158 MiB data, 443 MiB used, 715 GiB / 715 GiB avail; 6.7 KiB/s rd, 6.5 MiB/s wr, 598 op/s 2024-09-21T04:09:53.285 INFO:teuthology.orchestra.run.smithi089.stdout: 5 12901 13.23 MB/sec warmup 32 sec latency 52.770 ms 2024-09-21T04:09:53.358 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:09:53 smithi195 ceph-mon[25840]: pgmap v257: 97 pgs: 97 active+clean; 158 MiB data, 443 MiB used, 715 GiB / 715 GiB avail; 6.7 KiB/s rd, 6.5 MiB/s wr, 598 op/s 2024-09-21T04:09:54.285 INFO:teuthology.orchestra.run.smithi089.stdout: 5 13322 13.27 MB/sec warmup 33 sec latency 57.144 ms 2024-09-21T04:09:55.241 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:09:55 smithi089 ceph-mon[23733]: pgmap v258: 97 pgs: 97 active+clean; 154 MiB data, 462 MiB used, 715 GiB / 715 GiB avail; 6.7 KiB/s rd, 5.5 MiB/s wr, 507 op/s 2024-09-21T04:09:55.285 INFO:teuthology.orchestra.run.smithi089.stdout: 5 13912 13.63 MB/sec warmup 34 sec latency 48.340 ms 2024-09-21T04:09:55.358 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:09:55 smithi195 ceph-mon[25840]: pgmap v258: 97 pgs: 97 active+clean; 154 MiB data, 462 MiB used, 715 GiB / 715 GiB avail; 6.7 KiB/s rd, 5.5 MiB/s wr, 507 op/s 2024-09-21T04:09:56.285 INFO:teuthology.orchestra.run.smithi089.stdout: 5 14376 13.80 MB/sec warmup 35 sec latency 46.116 ms 2024-09-21T04:09:57.260 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:09:57 smithi089 ceph-mon[23733]: pgmap v259: 97 pgs: 97 active+clean; 154 MiB data, 469 MiB used, 715 GiB / 715 GiB avail; 11 KiB/s rd, 5.0 MiB/s wr, 487 op/s 2024-09-21T04:09:57.285 INFO:teuthology.orchestra.run.smithi089.stdout: 5 14909 13.56 MB/sec warmup 36 sec latency 33.046 ms 2024-09-21T04:09:57.358 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:09:57 smithi195 ceph-mon[25840]: pgmap v259: 97 pgs: 97 active+clean; 154 MiB data, 469 MiB used, 715 GiB / 715 GiB avail; 11 KiB/s rd, 5.0 MiB/s wr, 487 op/s 2024-09-21T04:09:58.286 INFO:teuthology.orchestra.run.smithi089.stdout: 5 15207 13.35 MB/sec warmup 37 sec latency 59.338 ms 2024-09-21T04:09:59.286 INFO:teuthology.orchestra.run.smithi089.stdout: 5 15603 13.41 MB/sec warmup 38 sec latency 93.081 ms 2024-09-21T04:09:59.348 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:09:59 smithi089 ceph-mon[23733]: pgmap v260: 97 pgs: 97 active+clean; 164 MiB data, 541 MiB used, 715 GiB / 715 GiB avail; 6.8 KiB/s rd, 7.3 MiB/s wr, 705 op/s 2024-09-21T04:09:59.358 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:09:59 smithi195 ceph-mon[25840]: pgmap v260: 97 pgs: 97 active+clean; 164 MiB data, 541 MiB used, 715 GiB / 715 GiB avail; 6.8 KiB/s rd, 7.3 MiB/s wr, 705 op/s 2024-09-21T04:10:00.122 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:10:00 smithi089 ceph-mon[23733]: overall HEALTH_OK 2024-09-21T04:10:00.286 INFO:teuthology.orchestra.run.smithi089.stdout: 5 16049 13.18 MB/sec warmup 39 sec latency 52.362 ms 2024-09-21T04:10:00.358 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:10:00 smithi195 ceph-mon[25840]: overall HEALTH_OK 2024-09-21T04:10:01.126 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:10:01 smithi089 ceph-mon[23733]: pgmap v261: 97 pgs: 97 active+clean; 170 MiB data, 546 MiB used, 715 GiB / 715 GiB avail; 9.3 KiB/s rd, 5.8 MiB/s wr, 561 op/s 2024-09-21T04:10:01.286 INFO:teuthology.orchestra.run.smithi089.stdout: 5 16555 13.02 MB/sec warmup 40 sec latency 52.196 ms 2024-09-21T04:10:01.358 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:10:01 smithi195 ceph-mon[25840]: pgmap v261: 97 pgs: 97 active+clean; 170 MiB data, 546 MiB used, 715 GiB / 715 GiB avail; 9.3 KiB/s rd, 5.8 MiB/s wr, 561 op/s 2024-09-21T04:10:02.161 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:10:02 smithi089 ceph-mon[23733]: pgmap v262: 97 pgs: 97 active+clean; 169 MiB data, 586 MiB used, 715 GiB / 715 GiB avail; 7.2 KiB/s rd, 6.7 MiB/s wr, 675 op/s 2024-09-21T04:10:02.292 INFO:teuthology.orchestra.run.smithi089.stdout: 5 17052 13.26 MB/sec warmup 41 sec latency 67.623 ms 2024-09-21T04:10:02.358 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:10:02 smithi195 ceph-mon[25840]: pgmap v262: 97 pgs: 97 active+clean; 169 MiB data, 586 MiB used, 715 GiB / 715 GiB avail; 7.2 KiB/s rd, 6.7 MiB/s wr, 675 op/s 2024-09-21T04:10:03.292 INFO:teuthology.orchestra.run.smithi089.stdout: 5 17641 13.51 MB/sec warmup 42 sec latency 55.046 ms 2024-09-21T04:10:04.196 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:10:04 smithi089 ceph-mon[23733]: pgmap v263: 97 pgs: 97 active+clean; 166 MiB data, 592 MiB used, 715 GiB / 715 GiB avail; 7.2 KiB/s rd, 6.1 MiB/s wr, 610 op/s 2024-09-21T04:10:04.292 INFO:teuthology.orchestra.run.smithi089.stdout: 5 17965 13.49 MB/sec warmup 43 sec latency 72.603 ms 2024-09-21T04:10:04.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:10:04 smithi195 ceph-mon[25840]: pgmap v263: 97 pgs: 97 active+clean; 166 MiB data, 592 MiB used, 715 GiB / 715 GiB avail; 7.2 KiB/s rd, 6.1 MiB/s wr, 610 op/s 2024-09-21T04:10:05.292 INFO:teuthology.orchestra.run.smithi089.stdout: 5 18169 13.23 MB/sec warmup 44 sec latency 73.425 ms 2024-09-21T04:10:06.250 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:10:06 smithi089 ceph-mon[23733]: pgmap v264: 97 pgs: 97 active+clean; 166 MiB data, 579 MiB used, 715 GiB / 715 GiB avail; 11 KiB/s rd, 5.3 MiB/s wr, 530 op/s 2024-09-21T04:10:06.292 INFO:teuthology.orchestra.run.smithi089.stdout: 5 18719 13.07 MB/sec warmup 45 sec latency 54.486 ms 2024-09-21T04:10:06.410 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:10:06 smithi195 ceph-mon[25840]: pgmap v264: 97 pgs: 97 active+clean; 166 MiB data, 579 MiB used, 715 GiB / 715 GiB avail; 11 KiB/s rd, 5.3 MiB/s wr, 530 op/s 2024-09-21T04:10:07.293 INFO:teuthology.orchestra.run.smithi089.stdout: 5 19320 13.23 MB/sec warmup 46 sec latency 32.804 ms 2024-09-21T04:10:08.259 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:10:08 smithi089 ceph-mon[23733]: pgmap v265: 97 pgs: 97 active+clean; 180 MiB data, 648 MiB used, 715 GiB / 715 GiB avail; 7.2 KiB/s rd, 7.4 MiB/s wr, 669 op/s 2024-09-21T04:10:08.292 INFO:teuthology.orchestra.run.smithi089.stdout: 5 20010 13.05 MB/sec warmup 47 sec latency 29.217 ms 2024-09-21T04:10:08.608 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:10:08 smithi195 ceph-mon[25840]: pgmap v265: 97 pgs: 97 active+clean; 180 MiB data, 648 MiB used, 715 GiB / 715 GiB avail; 7.2 KiB/s rd, 7.4 MiB/s wr, 669 op/s 2024-09-21T04:10:09.293 INFO:teuthology.orchestra.run.smithi089.stdout: 5 20429 13.10 MB/sec warmup 48 sec latency 75.794 ms 2024-09-21T04:10:10.200 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:10:10 smithi089 ceph-mon[23733]: pgmap v266: 97 pgs: 97 active+clean; 190 MiB data, 683 MiB used, 715 GiB / 715 GiB avail; 12 KiB/s rd, 6.1 MiB/s wr, 653 op/s 2024-09-21T04:10:10.292 INFO:teuthology.orchestra.run.smithi089.stdout: 5 20958 13.27 MB/sec warmup 49 sec latency 32.476 ms 2024-09-21T04:10:10.594 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:10:10 smithi195 ceph-mon[25840]: pgmap v266: 97 pgs: 97 active+clean; 190 MiB data, 683 MiB used, 715 GiB / 715 GiB avail; 12 KiB/s rd, 6.1 MiB/s wr, 653 op/s 2024-09-21T04:10:11.293 INFO:teuthology.orchestra.run.smithi089.stdout: 5 21444 13.45 MB/sec warmup 50 sec latency 48.366 ms 2024-09-21T04:10:12.293 INFO:teuthology.orchestra.run.smithi089.stdout: 5 21633 13.28 MB/sec warmup 51 sec latency 63.637 ms 2024-09-21T04:10:12.379 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:10:12 smithi089 ceph-mon[23733]: pgmap v267: 97 pgs: 97 active+clean; 190 MiB data, 719 MiB used, 715 GiB / 715 GiB avail; 9.2 KiB/s rd, 7.0 MiB/s wr, 648 op/s 2024-09-21T04:10:12.608 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:10:12 smithi195 ceph-mon[25840]: pgmap v267: 97 pgs: 97 active+clean; 190 MiB data, 719 MiB used, 715 GiB / 715 GiB avail; 9.2 KiB/s rd, 7.0 MiB/s wr, 648 op/s 2024-09-21T04:10:13.293 INFO:teuthology.orchestra.run.smithi089.stdout: 5 21947 13.06 MB/sec warmup 52 sec latency 51.726 ms 2024-09-21T04:10:13.387 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:10:13 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi089.srlyxf/mirror_snapshot_schedule"}]: dispatch 2024-09-21T04:10:13.387 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:10:13 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi089.srlyxf/trash_purge_schedule"}]: dispatch 2024-09-21T04:10:13.608 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:10:13 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi089.srlyxf/mirror_snapshot_schedule"}]: dispatch 2024-09-21T04:10:13.608 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:10:13 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi089.srlyxf/trash_purge_schedule"}]: dispatch 2024-09-21T04:10:14.293 INFO:teuthology.orchestra.run.smithi089.stdout: 5 22308 12.91 MB/sec warmup 53 sec latency 38.217 ms 2024-09-21T04:10:14.401 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:10:14 smithi089 ceph-mon[23733]: pgmap v268: 97 pgs: 97 active+clean; 188 MiB data, 723 MiB used, 715 GiB / 715 GiB avail; 9.2 KiB/s rd, 6.6 MiB/s wr, 554 op/s 2024-09-21T04:10:14.608 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:10:14 smithi195 ceph-mon[25840]: pgmap v268: 97 pgs: 97 active+clean; 188 MiB data, 723 MiB used, 715 GiB / 715 GiB avail; 9.2 KiB/s rd, 6.6 MiB/s wr, 554 op/s 2024-09-21T04:10:15.293 INFO:teuthology.orchestra.run.smithi089.stdout: 5 22711 12.92 MB/sec warmup 54 sec latency 50.977 ms 2024-09-21T04:10:16.192 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:10:16 smithi089 ceph-mon[23733]: pgmap v269: 97 pgs: 97 active+clean; 191 MiB data, 743 MiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 6.2 MiB/s wr, 558 op/s 2024-09-21T04:10:16.293 INFO:teuthology.orchestra.run.smithi089.stdout: 5 23303 12.84 MB/sec warmup 55 sec latency 48.092 ms 2024-09-21T04:10:16.608 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:10:16 smithi195 ceph-mon[25840]: pgmap v269: 97 pgs: 97 active+clean; 191 MiB data, 743 MiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 6.2 MiB/s wr, 558 op/s 2024-09-21T04:10:17.293 INFO:teuthology.orchestra.run.smithi089.stdout: 5 23712 12.74 MB/sec warmup 56 sec latency 46.041 ms 2024-09-21T04:10:17.749 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:10:17 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' 2024-09-21T04:10:17.749 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:10:17 smithi089 ceph-mon[23733]: Upgrade: Target is version 19.3.0-5103-gf36283e7 (unknown) 2024-09-21T04:10:17.749 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:10:17 smithi089 ceph-mon[23733]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:2195e5f178be0a9390728e2a14088236d8fe6f82635f5c84418fe37a04457e00, digests ['quay.ceph.io/ceph-ci/ceph@sha256:2195e5f178be0a9390728e2a14088236d8fe6f82635f5c84418fe37a04457e00'] 2024-09-21T04:10:17.749 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:10:17 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T04:10:17.749 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:10:17 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' 2024-09-21T04:10:17.749 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:10:17 smithi089 ceph-mon[23733]: Upgrade: Need to upgrade myself (mgr.smithi089.srlyxf) 2024-09-21T04:10:17.749 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:10:17 smithi089 ceph-mon[23733]: Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:2195e5f178be0a9390728e2a14088236d8fe6f82635f5c84418fe37a04457e00 on smithi195 2024-09-21T04:10:18.108 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:10:17 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' 2024-09-21T04:10:18.109 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:10:17 smithi195 ceph-mon[25840]: Upgrade: Target is version 19.3.0-5103-gf36283e7 (unknown) 2024-09-21T04:10:18.109 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:10:17 smithi195 ceph-mon[25840]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:2195e5f178be0a9390728e2a14088236d8fe6f82635f5c84418fe37a04457e00, digests ['quay.ceph.io/ceph-ci/ceph@sha256:2195e5f178be0a9390728e2a14088236d8fe6f82635f5c84418fe37a04457e00'] 2024-09-21T04:10:18.109 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:10:17 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T04:10:18.109 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:10:17 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' 2024-09-21T04:10:18.109 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:10:17 smithi195 ceph-mon[25840]: Upgrade: Need to upgrade myself (mgr.smithi089.srlyxf) 2024-09-21T04:10:18.109 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:10:17 smithi195 ceph-mon[25840]: Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:2195e5f178be0a9390728e2a14088236d8fe6f82635f5c84418fe37a04457e00 on smithi195 2024-09-21T04:10:18.293 INFO:teuthology.orchestra.run.smithi089.stdout: 5 24052 12.79 MB/sec warmup 57 sec latency 62.029 ms 2024-09-21T04:10:18.782 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:10:18 smithi089 ceph-mon[23733]: pgmap v270: 97 pgs: 97 active+clean; 191 MiB data, 789 MiB used, 714 GiB / 715 GiB avail; 7.8 KiB/s rd, 7.8 MiB/s wr, 750 op/s 2024-09-21T04:10:19.109 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:10:18 smithi195 ceph-mon[25840]: pgmap v270: 97 pgs: 97 active+clean; 191 MiB data, 789 MiB used, 714 GiB / 715 GiB avail; 7.8 KiB/s rd, 7.8 MiB/s wr, 750 op/s 2024-09-21T04:10:19.293 INFO:teuthology.orchestra.run.smithi089.stdout: 5 24587 12.89 MB/sec warmup 58 sec latency 71.208 ms 2024-09-21T04:10:20.293 INFO:teuthology.orchestra.run.smithi089.stdout: 5 25079 13.06 MB/sec warmup 59 sec latency 191.541 ms 2024-09-21T04:10:20.510 INFO:teuthology.orchestra.run.smithi089.stdout:true 2024-09-21T04:10:20.858 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:10:20 smithi195 ceph-mon[25840]: pgmap v271: 97 pgs: 97 active+clean; 189 MiB data, 796 MiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 5.7 MiB/s wr, 611 op/s 2024-09-21T04:10:20.898 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:10:20 smithi089 ceph-mon[23733]: pgmap v271: 97 pgs: 97 active+clean; 189 MiB data, 796 MiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 5.7 MiB/s wr, 611 op/s 2024-09-21T04:10:21.293 INFO:teuthology.orchestra.run.smithi089.stdout: 5 25481 12.92 MB/sec warmup 60 sec latency 36.102 ms 2024-09-21T04:10:21.480 INFO:teuthology.orchestra.run.smithi089.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-21T04:10:21.480 INFO:teuthology.orchestra.run.smithi089.stdout:alertmanager.smithi089 smithi089 *:9093,9094 running (6m) 62s ago 7m 22.6M - ba2b418f427c 703c928c9523 2024-09-21T04:10:21.481 INFO:teuthology.orchestra.run.smithi089.stdout:crash.smithi089 smithi089 running (7m) 62s ago 7m 6409k - 17.2.0 e1d6a67b021e d1e0a8969f78 2024-09-21T04:10:21.481 INFO:teuthology.orchestra.run.smithi089.stdout:crash.smithi195 smithi195 running (6m) 64s ago 6m 6400k - 17.2.0 e1d6a67b021e 7e213ff76084 2024-09-21T04:10:21.481 INFO:teuthology.orchestra.run.smithi089.stdout:grafana.smithi089 smithi089 *:3000 running (5m) 62s ago 7m 53.2M - 8.3.5 dad864ee21e9 547de193a0ee 2024-09-21T04:10:21.481 INFO:teuthology.orchestra.run.smithi089.stdout:mds.foofs.smithi089.osufws smithi089 running (95s) 62s ago 95s 13.3M - 17.2.0 e1d6a67b021e 664d41859bea 2024-09-21T04:10:21.481 INFO:teuthology.orchestra.run.smithi089.stdout:mds.foofs.smithi195.hijxre smithi195 running (93s) 64s ago 92s 14.2M - 17.2.0 e1d6a67b021e 7bc0c6542d12 2024-09-21T04:10:21.481 INFO:teuthology.orchestra.run.smithi089.stdout:mgr.smithi089.srlyxf smithi089 *:9283 running (8m) 62s ago 8m 461M - 17.2.0 e1d6a67b021e 410bb00e916d 2024-09-21T04:10:21.481 INFO:teuthology.orchestra.run.smithi089.stdout:mgr.smithi195.cgotzm smithi195 *:8443,9283 running (6m) 64s ago 6m 412M - 17.2.0 e1d6a67b021e f2b03f4a2ef8 2024-09-21T04:10:21.481 INFO:teuthology.orchestra.run.smithi089.stdout:mon.smithi089 smithi089 running (8m) 62s ago 8m 46.7M 2048M 17.2.0 e1d6a67b021e 1d2b3719e46e 2024-09-21T04:10:21.481 INFO:teuthology.orchestra.run.smithi089.stdout:mon.smithi195 smithi195 running (6m) 64s ago 6m 37.7M 2048M 17.2.0 e1d6a67b021e 7f86c5b5d21e 2024-09-21T04:10:21.481 INFO:teuthology.orchestra.run.smithi089.stdout:nfs.foo.0.0.smithi089.dcjmqc smithi089 *:2049 running (73s) 62s ago 72s 46.8M - 4.0 e1d6a67b021e 98550f94e773 2024-09-21T04:10:21.481 INFO:teuthology.orchestra.run.smithi089.stdout:nfs.foo.1.0.smithi195.vhoayp smithi195 *:2049 running (67s) 64s ago 67s 16.0M - 4.0 e1d6a67b021e c60a4040130a 2024-09-21T04:10:21.481 INFO:teuthology.orchestra.run.smithi089.stdout:node-exporter.smithi089 smithi089 *:9100 running (7m) 62s ago 7m 17.5M - 1dbe0e931976 5b166976fc1c 2024-09-21T04:10:21.481 INFO:teuthology.orchestra.run.smithi089.stdout:node-exporter.smithi195 smithi195 *:9100 running (6m) 64s ago 6m 17.0M - 1dbe0e931976 3448624c239b 2024-09-21T04:10:21.481 INFO:teuthology.orchestra.run.smithi089.stdout:osd.0 smithi089 running (6m) 62s ago 6m 42.6M 3215M 17.2.0 e1d6a67b021e db7c0e5020e7 2024-09-21T04:10:21.481 INFO:teuthology.orchestra.run.smithi089.stdout:osd.1 smithi089 running (5m) 62s ago 5m 43.6M 3215M 17.2.0 e1d6a67b021e f9ec697d08d0 2024-09-21T04:10:21.482 INFO:teuthology.orchestra.run.smithi089.stdout:osd.2 smithi089 running (5m) 62s ago 5m 40.8M 3215M 17.2.0 e1d6a67b021e cfa000948859 2024-09-21T04:10:21.482 INFO:teuthology.orchestra.run.smithi089.stdout:osd.3 smithi089 running (4m) 62s ago 4m 40.6M 3215M 17.2.0 e1d6a67b021e 31c610ed5815 2024-09-21T04:10:21.482 INFO:teuthology.orchestra.run.smithi089.stdout:osd.4 smithi195 running (4m) 64s ago 4m 39.3M 3984M 17.2.0 e1d6a67b021e f00e18a1f220 2024-09-21T04:10:21.482 INFO:teuthology.orchestra.run.smithi089.stdout:osd.5 smithi195 running (3m) 64s ago 3m 40.0M 3984M 17.2.0 e1d6a67b021e cecfd6ff1eee 2024-09-21T04:10:21.482 INFO:teuthology.orchestra.run.smithi089.stdout:osd.6 smithi195 running (3m) 64s ago 3m 39.1M 3984M 17.2.0 e1d6a67b021e 0d7db362cc23 2024-09-21T04:10:21.482 INFO:teuthology.orchestra.run.smithi089.stdout:osd.7 smithi195 running (3m) 64s ago 3m 40.5M 3984M 17.2.0 e1d6a67b021e 0109b3fea188 2024-09-21T04:10:21.482 INFO:teuthology.orchestra.run.smithi089.stdout:prometheus.smithi089 smithi089 *:9095 running (6m) 62s ago 6m 62.8M - 514e6a882f6e 9ed6ce8f2de3 2024-09-21T04:10:21.907 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:10:21 smithi089 ceph-mon[23733]: from='client.14654 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T04:10:21.907 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:10:21 smithi089 ceph-mon[23733]: from='client.14658 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T04:10:21.983 INFO:teuthology.orchestra.run.smithi089.stdout:{ 2024-09-21T04:10:21.983 INFO:teuthology.orchestra.run.smithi089.stdout: "mon": { 2024-09-21T04:10:21.983 INFO:teuthology.orchestra.run.smithi089.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-21T04:10:21.983 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2024-09-21T04:10:21.983 INFO:teuthology.orchestra.run.smithi089.stdout: "mgr": { 2024-09-21T04:10:21.983 INFO:teuthology.orchestra.run.smithi089.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-21T04:10:21.984 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2024-09-21T04:10:21.984 INFO:teuthology.orchestra.run.smithi089.stdout: "osd": { 2024-09-21T04:10:21.984 INFO:teuthology.orchestra.run.smithi089.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-09-21T04:10:21.984 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2024-09-21T04:10:21.984 INFO:teuthology.orchestra.run.smithi089.stdout: "mds": { 2024-09-21T04:10:21.984 INFO:teuthology.orchestra.run.smithi089.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-21T04:10:21.984 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2024-09-21T04:10:21.984 INFO:teuthology.orchestra.run.smithi089.stdout: "overall": { 2024-09-21T04:10:21.984 INFO:teuthology.orchestra.run.smithi089.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 14 2024-09-21T04:10:21.984 INFO:teuthology.orchestra.run.smithi089.stdout: } 2024-09-21T04:10:21.984 INFO:teuthology.orchestra.run.smithi089.stdout:} 2024-09-21T04:10:22.108 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:10:21 smithi195 ceph-mon[25840]: from='client.14654 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T04:10:22.109 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:10:21 smithi195 ceph-mon[25840]: from='client.14658 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T04:10:22.293 INFO:teuthology.orchestra.run.smithi089.stdout: 5 26002 12.87 MB/sec warmup 61 sec latency 56.070 ms 2024-09-21T04:10:22.388 INFO:teuthology.orchestra.run.smithi089.stdout:{ 2024-09-21T04:10:22.388 INFO:teuthology.orchestra.run.smithi089.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:2195e5f178be0a9390728e2a14088236d8fe6f82635f5c84418fe37a04457e00", 2024-09-21T04:10:22.388 INFO:teuthology.orchestra.run.smithi089.stdout: "in_progress": true, 2024-09-21T04:10:22.388 INFO:teuthology.orchestra.run.smithi089.stdout: "services_complete": [], 2024-09-21T04:10:22.388 INFO:teuthology.orchestra.run.smithi089.stdout: "progress": "0/23 daemons upgraded", 2024-09-21T04:10:22.388 INFO:teuthology.orchestra.run.smithi089.stdout: "message": "Pulling quay.ceph.io/ceph-ci/ceph@sha256:2195e5f178be0a9390728e2a14088236d8fe6f82635f5c84418fe37a04457e00 image on host smithi195" 2024-09-21T04:10:22.388 INFO:teuthology.orchestra.run.smithi089.stdout:} 2024-09-21T04:10:22.910 INFO:teuthology.orchestra.run.smithi089.stdout:HEALTH_OK 2024-09-21T04:10:22.913 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:10:22 smithi089 ceph-mon[23733]: pgmap v272: 97 pgs: 97 active+clean; 199 MiB data, 814 MiB used, 714 GiB / 715 GiB avail; 7.1 KiB/s rd, 6.5 MiB/s wr, 553 op/s 2024-09-21T04:10:22.913 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:10:22 smithi089 ceph-mon[23733]: from='client.14662 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T04:10:22.913 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:10:22 smithi089 ceph-mon[23733]: from='client.? 172.21.15.89:0/1716276758' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T04:10:23.109 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:10:22 smithi195 ceph-mon[25840]: pgmap v272: 97 pgs: 97 active+clean; 199 MiB data, 814 MiB used, 714 GiB / 715 GiB avail; 7.1 KiB/s rd, 6.5 MiB/s wr, 553 op/s 2024-09-21T04:10:23.109 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:10:22 smithi195 ceph-mon[25840]: from='client.14662 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T04:10:23.109 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:10:22 smithi195 ceph-mon[25840]: from='client.? 172.21.15.89:0/1716276758' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T04:10:23.293 INFO:teuthology.orchestra.run.smithi089.stdout: 5 26594 12.91 MB/sec warmup 62 sec latency 32.240 ms 2024-09-21T04:10:23.931 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:10:23 smithi089 ceph-mon[23733]: from='client.14670 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T04:10:23.968 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:10:23 smithi089 ceph-mon[23733]: from='client.? 172.21.15.89:0/3157357894' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-21T04:10:24.108 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:10:23 smithi195 ceph-mon[25840]: from='client.14670 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T04:10:24.108 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:10:23 smithi195 ceph-mon[25840]: from='client.? 172.21.15.89:0/3157357894' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-21T04:10:24.293 INFO:teuthology.orchestra.run.smithi089.stdout: 5 27136 12.81 MB/sec warmup 63 sec latency 30.826 ms 2024-09-21T04:10:24.953 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:10:24 smithi089 ceph-mon[23733]: pgmap v273: 97 pgs: 97 active+clean; 211 MiB data, 833 MiB used, 714 GiB / 715 GiB avail; 7.1 KiB/s rd, 6.6 MiB/s wr, 642 op/s 2024-09-21T04:10:25.108 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:10:24 smithi195 ceph-mon[25840]: pgmap v273: 97 pgs: 97 active+clean; 211 MiB data, 833 MiB used, 714 GiB / 715 GiB avail; 7.1 KiB/s rd, 6.6 MiB/s wr, 642 op/s 2024-09-21T04:10:25.293 INFO:teuthology.orchestra.run.smithi089.stdout: 5 27515 12.81 MB/sec warmup 64 sec latency 51.890 ms 2024-09-21T04:10:26.293 INFO:teuthology.orchestra.run.smithi089.stdout: 5 27936 12.90 MB/sec warmup 65 sec latency 48.657 ms 2024-09-21T04:10:27.020 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:10:26 smithi089 ceph-mon[23733]: pgmap v274: 97 pgs: 97 active+clean; 209 MiB data, 857 MiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 5.9 MiB/s wr, 647 op/s 2024-09-21T04:10:27.108 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:10:26 smithi195 ceph-mon[25840]: pgmap v274: 97 pgs: 97 active+clean; 209 MiB data, 857 MiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 5.9 MiB/s wr, 647 op/s 2024-09-21T04:10:27.293 INFO:teuthology.orchestra.run.smithi089.stdout: 5 28405 12.98 MB/sec warmup 66 sec latency 58.999 ms 2024-09-21T04:10:28.293 INFO:teuthology.orchestra.run.smithi089.stdout: 5 28653 12.93 MB/sec warmup 67 sec latency 48.514 ms 2024-09-21T04:10:28.795 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:10:28 smithi089 ceph-mon[23733]: pgmap v275: 97 pgs: 97 active+clean; 211 MiB data, 906 MiB used, 714 GiB / 715 GiB avail; 9.0 KiB/s rd, 7.6 MiB/s wr, 738 op/s 2024-09-21T04:10:29.108 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:10:28 smithi195 ceph-mon[25840]: pgmap v275: 97 pgs: 97 active+clean; 211 MiB data, 906 MiB used, 714 GiB / 715 GiB avail; 9.0 KiB/s rd, 7.6 MiB/s wr, 738 op/s 2024-09-21T04:10:29.298 INFO:teuthology.orchestra.run.smithi089.stdout: 5 29156 12.82 MB/sec warmup 68 sec latency 64.846 ms 2024-09-21T04:10:30.298 INFO:teuthology.orchestra.run.smithi089.stdout: 5 29983 12.94 MB/sec warmup 69 sec latency 27.274 ms 2024-09-21T04:10:30.813 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:10:30 smithi089 ceph-mon[23733]: pgmap v276: 97 pgs: 97 active+clean; 212 MiB data, 929 MiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 6.2 MiB/s wr, 603 op/s 2024-09-21T04:10:30.858 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:10:30 smithi195 ceph-mon[25840]: pgmap v276: 97 pgs: 97 active+clean; 212 MiB data, 929 MiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 6.2 MiB/s wr, 603 op/s 2024-09-21T04:10:31.298 INFO:teuthology.orchestra.run.smithi089.stdout: 5 31014 13.06 MB/sec warmup 70 sec latency 21.549 ms 2024-09-21T04:10:32.298 INFO:teuthology.orchestra.run.smithi089.stdout: 5 32082 13.49 MB/sec warmup 71 sec latency 20.011 ms 2024-09-21T04:10:32.978 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:10:32 smithi089 ceph-mon[23733]: pgmap v277: 97 pgs: 97 active+clean; 218 MiB data, 942 MiB used, 714 GiB / 715 GiB avail; 7.2 KiB/s rd, 7.5 MiB/s wr, 649 op/s 2024-09-21T04:10:33.108 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:10:32 smithi195 ceph-mon[25840]: pgmap v277: 97 pgs: 97 active+clean; 218 MiB data, 942 MiB used, 714 GiB / 715 GiB avail; 7.2 KiB/s rd, 7.5 MiB/s wr, 649 op/s 2024-09-21T04:10:33.298 INFO:teuthology.orchestra.run.smithi089.stdout: 5 32737 13.44 MB/sec warmup 72 sec latency 47.272 ms 2024-09-21T04:10:34.298 INFO:teuthology.orchestra.run.smithi089.stdout: 5 33463 13.50 MB/sec warmup 73 sec latency 31.301 ms 2024-09-21T04:10:35.014 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:10:34 smithi089 ceph-mon[23733]: pgmap v278: 97 pgs: 97 active+clean; 229 MiB data, 982 MiB used, 714 GiB / 715 GiB avail; 7.2 KiB/s rd, 7.8 MiB/s wr, 785 op/s 2024-09-21T04:10:35.108 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:10:34 smithi195 ceph-mon[25840]: pgmap v278: 97 pgs: 97 active+clean; 229 MiB data, 982 MiB used, 714 GiB / 715 GiB avail; 7.2 KiB/s rd, 7.8 MiB/s wr, 785 op/s 2024-09-21T04:10:35.298 INFO:teuthology.orchestra.run.smithi089.stdout: 5 34386 13.55 MB/sec warmup 74 sec latency 22.812 ms 2024-09-21T04:10:36.307 INFO:teuthology.orchestra.run.smithi089.stdout: 5 35029 13.72 MB/sec warmup 75 sec latency 209.144 ms 2024-09-21T04:10:37.047 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:10:36 smithi089 ceph-mon[23733]: pgmap v279: 97 pgs: 97 active+clean; 228 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 14 KiB/s rd, 6.6 MiB/s wr, 645 op/s 2024-09-21T04:10:37.108 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:10:36 smithi195 ceph-mon[25840]: pgmap v279: 97 pgs: 97 active+clean; 228 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 14 KiB/s rd, 6.6 MiB/s wr, 645 op/s 2024-09-21T04:10:37.306 INFO:teuthology.orchestra.run.smithi089.stdout: 5 35809 13.94 MB/sec warmup 76 sec latency 162.283 ms 2024-09-21T04:10:38.302 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:10:38 smithi089 ceph-mon[23733]: pgmap v280: 97 pgs: 97 active+clean; 238 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 10 KiB/s rd, 11 MiB/s wr, 979 op/s 2024-09-21T04:10:38.306 INFO:teuthology.orchestra.run.smithi089.stdout: 5 36471 13.85 MB/sec warmup 77 sec latency 12.364 ms 2024-09-21T04:10:38.608 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:10:38 smithi195 ceph-mon[25840]: pgmap v280: 97 pgs: 97 active+clean; 238 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 10 KiB/s rd, 11 MiB/s wr, 979 op/s 2024-09-21T04:10:39.307 INFO:teuthology.orchestra.run.smithi089.stdout: 5 37368 13.95 MB/sec warmup 78 sec latency 23.637 ms 2024-09-21T04:10:40.307 INFO:teuthology.orchestra.run.smithi089.stdout: 5 38154 14.01 MB/sec warmup 79 sec latency 17.750 ms 2024-09-21T04:10:40.315 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:10:40 smithi089 ceph-mon[23733]: pgmap v281: 97 pgs: 97 active+clean; 241 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 14 KiB/s rd, 9.1 MiB/s wr, 921 op/s 2024-09-21T04:10:40.595 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:10:40 smithi195 ceph-mon[25840]: pgmap v281: 97 pgs: 97 active+clean; 241 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 14 KiB/s rd, 9.1 MiB/s wr, 921 op/s 2024-09-21T04:10:41.307 INFO:teuthology.orchestra.run.smithi089.stdout: 5 38985 14.27 MB/sec warmup 80 sec latency 19.763 ms 2024-09-21T04:10:42.307 INFO:teuthology.orchestra.run.smithi089.stdout: 5 39671 14.29 MB/sec warmup 81 sec latency 19.620 ms 2024-09-21T04:10:42.330 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:10:42 smithi089 ceph-mon[23733]: pgmap v282: 97 pgs: 97 active+clean; 241 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 11 MiB/s wr, 937 op/s 2024-09-21T04:10:42.608 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:10:42 smithi195 ceph-mon[25840]: pgmap v282: 97 pgs: 97 active+clean; 241 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 11 MiB/s wr, 937 op/s 2024-09-21T04:10:43.307 INFO:teuthology.orchestra.run.smithi089.stdout: 5 40539 14.34 MB/sec warmup 82 sec latency 16.068 ms 2024-09-21T04:10:44.307 INFO:teuthology.orchestra.run.smithi089.stdout: 5 41570 14.43 MB/sec warmup 83 sec latency 25.301 ms 2024-09-21T04:10:44.340 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:10:44 smithi089 ceph-mon[23733]: pgmap v283: 97 pgs: 97 active+clean; 247 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 11 MiB/s wr, 1.05k op/s 2024-09-21T04:10:44.608 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:10:44 smithi195 ceph-mon[25840]: pgmap v283: 97 pgs: 97 active+clean; 247 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 11 MiB/s wr, 1.05k op/s 2024-09-21T04:10:45.307 INFO:teuthology.orchestra.run.smithi089.stdout: 5 42316 14.64 MB/sec warmup 84 sec latency 51.862 ms 2024-09-21T04:10:46.307 INFO:teuthology.orchestra.run.smithi089.stdout: 5 42960 14.72 MB/sec warmup 85 sec latency 16.456 ms 2024-09-21T04:10:46.353 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:10:46 smithi089 ceph-mon[23733]: pgmap v284: 97 pgs: 97 active+clean; 251 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 18 KiB/s rd, 10 MiB/s wr, 876 op/s 2024-09-21T04:10:46.358 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:10:46 smithi195 ceph-mon[25840]: pgmap v284: 97 pgs: 97 active+clean; 251 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 18 KiB/s rd, 10 MiB/s wr, 876 op/s 2024-09-21T04:10:47.307 INFO:teuthology.orchestra.run.smithi089.stdout: 5 43624 14.64 MB/sec warmup 86 sec latency 68.738 ms 2024-09-21T04:10:48.290 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:10:48 smithi089 ceph-mon[23733]: pgmap v285: 97 pgs: 97 active+clean; 265 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 14 MiB/s wr, 1.24k op/s 2024-09-21T04:10:48.307 INFO:teuthology.orchestra.run.smithi089.stdout: 5 44436 14.70 MB/sec warmup 87 sec latency 30.543 ms 2024-09-21T04:10:48.608 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:10:48 smithi195 ceph-mon[25840]: pgmap v285: 97 pgs: 97 active+clean; 265 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 14 MiB/s wr, 1.24k op/s 2024-09-21T04:10:49.307 INFO:teuthology.orchestra.run.smithi089.stdout: 5 45366 14.80 MB/sec warmup 88 sec latency 26.051 ms 2024-09-21T04:10:49.545 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:10:49 smithi089 ceph-mon[23733]: Upgrade: Updating mgr.smithi195.cgotzm 2024-09-21T04:10:49.545 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:10:49 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' 2024-09-21T04:10:49.545 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:10:49 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi195.cgotzm", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-21T04:10:49.545 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:10:49 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-21T04:10:49.545 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:10:49 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T04:10:49.545 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:10:49 smithi089 ceph-mon[23733]: Deploying daemon mgr.smithi195.cgotzm on smithi195 2024-09-21T04:10:49.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:10:49 smithi195 ceph-mon[25840]: Upgrade: Updating mgr.smithi195.cgotzm 2024-09-21T04:10:49.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:10:49 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' 2024-09-21T04:10:49.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:10:49 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi195.cgotzm", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-21T04:10:49.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:10:49 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-21T04:10:49.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:10:49 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T04:10:49.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:10:49 smithi195 ceph-mon[25840]: Deploying daemon mgr.smithi195.cgotzm on smithi195 2024-09-21T04:10:50.307 INFO:teuthology.orchestra.run.smithi089.stdout: 5 46262 15.03 MB/sec warmup 89 sec latency 14.816 ms 2024-09-21T04:10:50.525 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:10:50 smithi195 ceph-mon[25840]: pgmap v286: 97 pgs: 97 active+clean; 266 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 18 KiB/s rd, 10 MiB/s wr, 917 op/s 2024-09-21T04:10:50.548 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:10:50 smithi089 ceph-mon[23733]: pgmap v286: 97 pgs: 97 active+clean; 266 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 18 KiB/s rd, 10 MiB/s wr, 917 op/s 2024-09-21T04:10:51.307 INFO:teuthology.orchestra.run.smithi089.stdout: 5 46999 15.02 MB/sec warmup 90 sec latency 18.462 ms 2024-09-21T04:10:52.307 INFO:teuthology.orchestra.run.smithi089.stdout: 5 47852 15.04 MB/sec warmup 91 sec latency 29.183 ms 2024-09-21T04:10:52.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:10:52 smithi195 ceph-mon[25840]: pgmap v287: 97 pgs: 97 active+clean; 274 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 14 KiB/s rd, 12 MiB/s wr, 1.00k op/s 2024-09-21T04:10:52.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:10:52 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' 2024-09-21T04:10:52.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:10:52 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T04:10:52.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:10:52 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T04:10:52.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:10:52 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T04:10:52.562 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:10:52 smithi089 ceph-mon[23733]: pgmap v287: 97 pgs: 97 active+clean; 274 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 14 KiB/s rd, 12 MiB/s wr, 1.00k op/s 2024-09-21T04:10:52.564 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:10:52 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' 2024-09-21T04:10:52.564 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:10:52 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T04:10:52.564 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:10:52 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T04:10:52.564 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:10:52 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T04:10:53.307 INFO:teuthology.orchestra.run.smithi089.stdout: 5 48659 15.10 MB/sec warmup 92 sec latency 23.716 ms 2024-09-21T04:10:53.455 INFO:teuthology.orchestra.run.smithi089.stdout:true 2024-09-21T04:10:54.271 INFO:teuthology.orchestra.run.smithi089.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-21T04:10:54.271 INFO:teuthology.orchestra.run.smithi089.stdout:alertmanager.smithi089 smithi089 *:9093,9094 running (6m) 95s ago 8m 22.6M - ba2b418f427c 703c928c9523 2024-09-21T04:10:54.271 INFO:teuthology.orchestra.run.smithi089.stdout:crash.smithi089 smithi089 running (8m) 95s ago 8m 6409k - 17.2.0 e1d6a67b021e d1e0a8969f78 2024-09-21T04:10:54.271 INFO:teuthology.orchestra.run.smithi089.stdout:crash.smithi195 smithi195 running (7m) 97s ago 7m 6400k - 17.2.0 e1d6a67b021e 7e213ff76084 2024-09-21T04:10:54.271 INFO:teuthology.orchestra.run.smithi089.stdout:grafana.smithi089 smithi089 *:3000 running (6m) 95s ago 7m 53.2M - 8.3.5 dad864ee21e9 547de193a0ee 2024-09-21T04:10:54.271 INFO:teuthology.orchestra.run.smithi089.stdout:mds.foofs.smithi089.osufws smithi089 running (2m) 95s ago 2m 13.3M - 17.2.0 e1d6a67b021e 664d41859bea 2024-09-21T04:10:54.271 INFO:teuthology.orchestra.run.smithi089.stdout:mds.foofs.smithi195.hijxre smithi195 running (2m) 97s ago 2m 14.2M - 17.2.0 e1d6a67b021e 7bc0c6542d12 2024-09-21T04:10:54.271 INFO:teuthology.orchestra.run.smithi089.stdout:mgr.smithi089.srlyxf smithi089 *:9283 running (9m) 95s ago 9m 461M - 17.2.0 e1d6a67b021e 410bb00e916d 2024-09-21T04:10:54.271 INFO:teuthology.orchestra.run.smithi089.stdout:mgr.smithi195.cgotzm smithi195 *:8443,9283 starting - - - - 2024-09-21T04:10:54.271 INFO:teuthology.orchestra.run.smithi089.stdout:mon.smithi089 smithi089 running (9m) 95s ago 9m 46.7M 2048M 17.2.0 e1d6a67b021e 1d2b3719e46e 2024-09-21T04:10:54.272 INFO:teuthology.orchestra.run.smithi089.stdout:mon.smithi195 smithi195 running (7m) 97s ago 7m 37.7M 2048M 17.2.0 e1d6a67b021e 7f86c5b5d21e 2024-09-21T04:10:54.272 INFO:teuthology.orchestra.run.smithi089.stdout:nfs.foo.0.0.smithi089.dcjmqc smithi089 *:2049 running (106s) 95s ago 105s 46.8M - 4.0 e1d6a67b021e 98550f94e773 2024-09-21T04:10:54.272 INFO:teuthology.orchestra.run.smithi089.stdout:nfs.foo.1.0.smithi195.vhoayp smithi195 *:2049 running (100s) 97s ago 100s 16.0M - 4.0 e1d6a67b021e c60a4040130a 2024-09-21T04:10:54.272 INFO:teuthology.orchestra.run.smithi089.stdout:node-exporter.smithi089 smithi089 *:9100 running (7m) 95s ago 7m 17.5M - 1dbe0e931976 5b166976fc1c 2024-09-21T04:10:54.272 INFO:teuthology.orchestra.run.smithi089.stdout:node-exporter.smithi195 smithi195 *:9100 running (7m) 97s ago 7m 17.0M - 1dbe0e931976 3448624c239b 2024-09-21T04:10:54.272 INFO:teuthology.orchestra.run.smithi089.stdout:osd.0 smithi089 running (6m) 95s ago 6m 42.6M 3215M 17.2.0 e1d6a67b021e db7c0e5020e7 2024-09-21T04:10:54.272 INFO:teuthology.orchestra.run.smithi089.stdout:osd.1 smithi089 running (6m) 95s ago 6m 43.6M 3215M 17.2.0 e1d6a67b021e f9ec697d08d0 2024-09-21T04:10:54.272 INFO:teuthology.orchestra.run.smithi089.stdout:osd.2 smithi089 running (5m) 95s ago 5m 40.8M 3215M 17.2.0 e1d6a67b021e cfa000948859 2024-09-21T04:10:54.272 INFO:teuthology.orchestra.run.smithi089.stdout:osd.3 smithi089 running (5m) 95s ago 5m 40.6M 3215M 17.2.0 e1d6a67b021e 31c610ed5815 2024-09-21T04:10:54.272 INFO:teuthology.orchestra.run.smithi089.stdout:osd.4 smithi195 running (4m) 97s ago 4m 39.3M 3984M 17.2.0 e1d6a67b021e f00e18a1f220 2024-09-21T04:10:54.272 INFO:teuthology.orchestra.run.smithi089.stdout:osd.5 smithi195 running (4m) 97s ago 4m 40.0M 3984M 17.2.0 e1d6a67b021e cecfd6ff1eee 2024-09-21T04:10:54.272 INFO:teuthology.orchestra.run.smithi089.stdout:osd.6 smithi195 running (4m) 97s ago 4m 39.1M 3984M 17.2.0 e1d6a67b021e 0d7db362cc23 2024-09-21T04:10:54.272 INFO:teuthology.orchestra.run.smithi089.stdout:osd.7 smithi195 running (3m) 97s ago 3m 40.5M 3984M 17.2.0 e1d6a67b021e 0109b3fea188 2024-09-21T04:10:54.272 INFO:teuthology.orchestra.run.smithi089.stdout:prometheus.smithi089 smithi089 *:9095 running (7m) 95s ago 7m 62.8M - 514e6a882f6e 9ed6ce8f2de3 2024-09-21T04:10:54.307 INFO:teuthology.orchestra.run.smithi089.stdout: 5 49257 15.19 MB/sec warmup 93 sec latency 54.475 ms 2024-09-21T04:10:54.339 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:10:54 smithi089 ceph-mon[23733]: pgmap v288: 97 pgs: 97 active+clean; 281 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 14 KiB/s rd, 12 MiB/s wr, 1.19k op/s 2024-09-21T04:10:54.339 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:10:54 smithi089 ceph-mon[23733]: from='client.14682 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T04:10:54.339 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:10:54 smithi089 ceph-mon[23733]: from='client.14686 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T04:10:54.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:10:54 smithi195 ceph-mon[25840]: pgmap v288: 97 pgs: 97 active+clean; 281 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 14 KiB/s rd, 12 MiB/s wr, 1.19k op/s 2024-09-21T04:10:54.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:10:54 smithi195 ceph-mon[25840]: from='client.14682 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T04:10:54.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:10:54 smithi195 ceph-mon[25840]: from='client.14686 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T04:10:54.879 INFO:teuthology.orchestra.run.smithi089.stdout:{ 2024-09-21T04:10:54.881 INFO:teuthology.orchestra.run.smithi089.stdout: "mon": { 2024-09-21T04:10:54.881 INFO:teuthology.orchestra.run.smithi089.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-21T04:10:54.882 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2024-09-21T04:10:54.882 INFO:teuthology.orchestra.run.smithi089.stdout: "mgr": { 2024-09-21T04:10:54.882 INFO:teuthology.orchestra.run.smithi089.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-21T04:10:54.882 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2024-09-21T04:10:54.882 INFO:teuthology.orchestra.run.smithi089.stdout: "osd": { 2024-09-21T04:10:54.883 INFO:teuthology.orchestra.run.smithi089.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-09-21T04:10:54.883 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2024-09-21T04:10:54.883 INFO:teuthology.orchestra.run.smithi089.stdout: "mds": { 2024-09-21T04:10:54.883 INFO:teuthology.orchestra.run.smithi089.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-21T04:10:54.883 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2024-09-21T04:10:54.883 INFO:teuthology.orchestra.run.smithi089.stdout: "overall": { 2024-09-21T04:10:54.884 INFO:teuthology.orchestra.run.smithi089.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 14 2024-09-21T04:10:54.884 INFO:teuthology.orchestra.run.smithi089.stdout: } 2024-09-21T04:10:54.884 INFO:teuthology.orchestra.run.smithi089.stdout:} 2024-09-21T04:10:55.307 INFO:teuthology.orchestra.run.smithi089.stdout: 5 49988 15.32 MB/sec warmup 94 sec latency 66.092 ms 2024-09-21T04:10:55.342 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:10:55 smithi089 ceph-mon[23733]: from='client.14690 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T04:10:55.343 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:10:55 smithi089 ceph-mon[23733]: from='client.? 172.21.15.89:0/751200208' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T04:10:55.402 INFO:teuthology.orchestra.run.smithi089.stdout:{ 2024-09-21T04:10:55.402 INFO:teuthology.orchestra.run.smithi089.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:2195e5f178be0a9390728e2a14088236d8fe6f82635f5c84418fe37a04457e00", 2024-09-21T04:10:55.402 INFO:teuthology.orchestra.run.smithi089.stdout: "in_progress": true, 2024-09-21T04:10:55.402 INFO:teuthology.orchestra.run.smithi089.stdout: "services_complete": [], 2024-09-21T04:10:55.402 INFO:teuthology.orchestra.run.smithi089.stdout: "progress": "", 2024-09-21T04:10:55.402 INFO:teuthology.orchestra.run.smithi089.stdout: "message": "Currently upgrading mgr daemons" 2024-09-21T04:10:55.402 INFO:teuthology.orchestra.run.smithi089.stdout:} 2024-09-21T04:10:55.608 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:10:55 smithi195 ceph-mon[25840]: from='client.14690 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T04:10:55.608 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:10:55 smithi195 ceph-mon[25840]: from='client.? 172.21.15.89:0/751200208' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T04:10:56.008 INFO:teuthology.orchestra.run.smithi089.stdout:HEALTH_OK 2024-09-21T04:10:56.307 INFO:teuthology.orchestra.run.smithi089.stdout: 5 50562 15.24 MB/sec warmup 95 sec latency 30.356 ms 2024-09-21T04:10:56.349 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:10:56 smithi089 ceph-mon[23733]: pgmap v289: 97 pgs: 97 active+clean; 282 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 21 KiB/s rd, 10 MiB/s wr, 1.04k op/s 2024-09-21T04:10:56.350 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:10:56 smithi089 ceph-mon[23733]: from='client.24423 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T04:10:56.350 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:10:56 smithi089 ceph-mon[23733]: from='client.? 172.21.15.89:0/3518358280' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-21T04:10:56.608 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:10:56 smithi195 ceph-mon[25840]: pgmap v289: 97 pgs: 97 active+clean; 282 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 21 KiB/s rd, 10 MiB/s wr, 1.04k op/s 2024-09-21T04:10:56.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:10:56 smithi195 ceph-mon[25840]: from='client.24423 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T04:10:56.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:10:56 smithi195 ceph-mon[25840]: from='client.? 172.21.15.89:0/3518358280' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-21T04:10:57.307 INFO:teuthology.orchestra.run.smithi089.stdout: 5 51290 15.24 MB/sec warmup 96 sec latency 31.049 ms 2024-09-21T04:10:58.307 INFO:teuthology.orchestra.run.smithi089.stdout: 5 52185 15.30 MB/sec warmup 97 sec latency 21.603 ms 2024-09-21T04:10:58.359 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:10:58 smithi089 ceph-mon[23733]: pgmap v290: 97 pgs: 97 active+clean; 285 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 14 KiB/s rd, 13 MiB/s wr, 1.09k op/s 2024-09-21T04:10:58.608 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:10:58 smithi195 ceph-mon[25840]: pgmap v290: 97 pgs: 97 active+clean; 285 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 14 KiB/s rd, 13 MiB/s wr, 1.09k op/s 2024-09-21T04:10:59.307 INFO:teuthology.orchestra.run.smithi089.stdout: 5 53008 15.49 MB/sec warmup 98 sec latency 27.822 ms 2024-09-21T04:10:59.363 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:10:59 smithi089 ceph-mon[23733]: Standby manager daemon smithi195.cgotzm restarted 2024-09-21T04:10:59.364 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:10:59 smithi089 ceph-mon[23733]: Standby manager daemon smithi195.cgotzm started 2024-09-21T04:10:59.364 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:10:59 smithi089 ceph-mon[23733]: from='mgr.? 172.21.15.195:0/751858916' entity='mgr.smithi195.cgotzm' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi195.cgotzm/crt"}]: dispatch 2024-09-21T04:10:59.364 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:10:59 smithi089 ceph-mon[23733]: from='mgr.? 172.21.15.195:0/751858916' entity='mgr.smithi195.cgotzm' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-21T04:10:59.364 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:10:59 smithi089 ceph-mon[23733]: from='mgr.? 172.21.15.195:0/751858916' entity='mgr.smithi195.cgotzm' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi195.cgotzm/key"}]: dispatch 2024-09-21T04:10:59.364 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:10:59 smithi089 ceph-mon[23733]: from='mgr.? 172.21.15.195:0/751858916' entity='mgr.smithi195.cgotzm' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-21T04:10:59.608 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:10:59 smithi195 ceph-mon[25840]: Standby manager daemon smithi195.cgotzm restarted 2024-09-21T04:10:59.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:10:59 smithi195 ceph-mon[25840]: Standby manager daemon smithi195.cgotzm started 2024-09-21T04:10:59.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:10:59 smithi195 ceph-mon[25840]: from='mgr.? 172.21.15.195:0/751858916' entity='mgr.smithi195.cgotzm' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi195.cgotzm/crt"}]: dispatch 2024-09-21T04:10:59.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:10:59 smithi195 ceph-mon[25840]: from='mgr.? 172.21.15.195:0/751858916' entity='mgr.smithi195.cgotzm' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-21T04:10:59.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:10:59 smithi195 ceph-mon[25840]: from='mgr.? 172.21.15.195:0/751858916' entity='mgr.smithi195.cgotzm' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi195.cgotzm/key"}]: dispatch 2024-09-21T04:10:59.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:10:59 smithi195 ceph-mon[25840]: from='mgr.? 172.21.15.195:0/751858916' entity='mgr.smithi195.cgotzm' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-21T04:11:00.307 INFO:teuthology.orchestra.run.smithi089.stdout: 5 53718 15.54 MB/sec warmup 99 sec latency 26.884 ms 2024-09-21T04:11:00.368 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:11:00 smithi089 ceph-mon[23733]: pgmap v291: 97 pgs: 97 active+clean; 293 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 20 KiB/s rd, 9.8 MiB/s wr, 936 op/s 2024-09-21T04:11:00.369 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:11:00 smithi089 ceph-mon[23733]: mgrmap e22: smithi089.srlyxf(active, since 7m), standbys: smithi195.cgotzm 2024-09-21T04:11:00.601 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:11:00 smithi195 ceph-mon[25840]: pgmap v291: 97 pgs: 97 active+clean; 293 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 20 KiB/s rd, 9.8 MiB/s wr, 936 op/s 2024-09-21T04:11:00.601 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:11:00 smithi195 ceph-mon[25840]: mgrmap e22: smithi089.srlyxf(active, since 7m), standbys: smithi195.cgotzm 2024-09-21T04:11:01.307 INFO:teuthology.orchestra.run.smithi089.stdout: 5 54269 15.46 MB/sec warmup 100 sec latency 77.952 ms 2024-09-21T04:11:02.308 INFO:teuthology.orchestra.run.smithi089.stdout: 5 54940 15.47 MB/sec warmup 101 sec latency 149.935 ms 2024-09-21T04:11:02.379 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:11:02 smithi089 ceph-mon[23733]: pgmap v292: 97 pgs: 97 active+clean; 293 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 13 KiB/s rd, 11 MiB/s wr, 954 op/s 2024-09-21T04:11:02.608 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:11:02 smithi195 ceph-mon[25840]: pgmap v292: 97 pgs: 97 active+clean; 293 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 13 KiB/s rd, 11 MiB/s wr, 954 op/s 2024-09-21T04:11:03.308 INFO:teuthology.orchestra.run.smithi089.stdout: 5 55765 15.50 MB/sec warmup 102 sec latency 37.702 ms 2024-09-21T04:11:04.307 INFO:teuthology.orchestra.run.smithi089.stdout: 5 56680 15.71 MB/sec warmup 103 sec latency 96.281 ms 2024-09-21T04:11:04.474 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:11:04 smithi089 ceph-mon[23733]: pgmap v293: 97 pgs: 97 active+clean; 304 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 13 KiB/s rd, 11 MiB/s wr, 1.06k op/s 2024-09-21T04:11:04.608 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:11:04 smithi195 ceph-mon[25840]: pgmap v293: 97 pgs: 97 active+clean; 304 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 13 KiB/s rd, 11 MiB/s wr, 1.06k op/s 2024-09-21T04:11:05.308 INFO:teuthology.orchestra.run.smithi089.stdout: 5 57317 15.72 MB/sec warmup 104 sec latency 19.709 ms 2024-09-21T04:11:06.307 INFO:teuthology.orchestra.run.smithi089.stdout: 5 57958 15.69 MB/sec warmup 105 sec latency 60.378 ms 2024-09-21T04:11:06.546 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:11:06 smithi089 ceph-mon[23733]: pgmap v294: 97 pgs: 97 active+clean; 304 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 18 KiB/s rd, 8.9 MiB/s wr, 803 op/s 2024-09-21T04:11:06.608 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:11:06 smithi195 ceph-mon[25840]: pgmap v294: 97 pgs: 97 active+clean; 304 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 18 KiB/s rd, 8.9 MiB/s wr, 803 op/s 2024-09-21T04:11:07.308 INFO:teuthology.orchestra.run.smithi089.stdout: 5 58820 15.70 MB/sec warmup 106 sec latency 29.366 ms 2024-09-21T04:11:08.308 INFO:teuthology.orchestra.run.smithi089.stdout: 5 59409 15.69 MB/sec warmup 107 sec latency 52.521 ms 2024-09-21T04:11:09.065 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:11:08 smithi089 ceph-mon[23733]: pgmap v295: 97 pgs: 97 active+clean; 309 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 12 MiB/s wr, 1.02k op/s 2024-09-21T04:11:09.065 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:11:08 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' 2024-09-21T04:11:09.065 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:11:08 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' 2024-09-21T04:11:09.199 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:11:08 smithi195 ceph-mon[25840]: pgmap v295: 97 pgs: 97 active+clean; 309 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 12 MiB/s wr, 1.02k op/s 2024-09-21T04:11:09.199 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:11:08 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' 2024-09-21T04:11:09.199 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:11:08 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' 2024-09-21T04:11:09.308 INFO:teuthology.orchestra.run.smithi089.stdout: 5 59929 15.74 MB/sec warmup 108 sec latency 36.627 ms 2024-09-21T04:11:10.308 INFO:teuthology.orchestra.run.smithi089.stdout: 5 60305 15.74 MB/sec warmup 109 sec latency 71.884 ms 2024-09-21T04:11:11.156 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:11:10 smithi089 ceph-mon[23733]: pgmap v296: 97 pgs: 97 active+clean; 313 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 18 KiB/s rd, 9.6 MiB/s wr, 959 op/s 2024-09-21T04:11:11.308 INFO:teuthology.orchestra.run.smithi089.stdout: 5 60625 15.71 MB/sec warmup 110 sec latency 49.673 ms 2024-09-21T04:11:11.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:11:10 smithi195 ceph-mon[25840]: pgmap v296: 97 pgs: 97 active+clean; 313 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 18 KiB/s rd, 9.6 MiB/s wr, 959 op/s 2024-09-21T04:11:12.308 INFO:teuthology.orchestra.run.smithi089.stdout: 5 60901 15.59 MB/sec warmup 111 sec latency 54.345 ms 2024-09-21T04:11:13.223 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:11:12 smithi089 ceph-mon[23733]: pgmap v297: 97 pgs: 97 active+clean; 314 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 10 MiB/s wr, 813 op/s 2024-09-21T04:11:13.254 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:11:12 smithi195 ceph-mon[25840]: pgmap v297: 97 pgs: 97 active+clean; 314 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 10 MiB/s wr, 813 op/s 2024-09-21T04:11:13.308 INFO:teuthology.orchestra.run.smithi089.stdout: 5 61226 15.48 MB/sec warmup 112 sec latency 109.171 ms 2024-09-21T04:11:14.235 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:11:13 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi089.srlyxf/trash_purge_schedule"}]: dispatch 2024-09-21T04:11:14.236 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:11:13 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi089.srlyxf/mirror_snapshot_schedule"}]: dispatch 2024-09-21T04:11:14.236 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:11:13 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' 2024-09-21T04:11:14.236 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:11:13 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' 2024-09-21T04:11:14.236 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:11:13 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' 2024-09-21T04:11:14.236 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:11:13 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T04:11:14.236 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:11:13 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "mgr fail", "who": "smithi089.srlyxf"}]: dispatch 2024-09-21T04:11:14.236 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:11:13 smithi089 ceph-mon[23733]: osdmap e71: 8 total, 8 up, 8 in 2024-09-21T04:11:14.301 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:11:13 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi089.srlyxf/trash_purge_schedule"}]: dispatch 2024-09-21T04:11:14.301 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:11:13 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi089.srlyxf/mirror_snapshot_schedule"}]: dispatch 2024-09-21T04:11:14.301 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:11:13 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' 2024-09-21T04:11:14.302 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:11:13 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' 2024-09-21T04:11:14.302 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:11:13 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' 2024-09-21T04:11:14.302 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:11:13 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T04:11:14.302 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:11:13 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "mgr fail", "who": "smithi089.srlyxf"}]: dispatch 2024-09-21T04:11:14.302 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:11:13 smithi195 ceph-mon[25840]: osdmap e71: 8 total, 8 up, 8 in 2024-09-21T04:11:14.308 INFO:teuthology.orchestra.run.smithi089.stdout: 5 61680 15.48 MB/sec warmup 113 sec latency 127.267 ms 2024-09-21T04:11:14.993 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:11:14 smithi089 ceph-mon[23733]: pgmap v298: 97 pgs: 97 active+clean; 323 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 10 MiB/s wr, 1.00k op/s 2024-09-21T04:11:14.993 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:11:14 smithi089 ceph-mon[23733]: Upgrade: Need to upgrade myself (mgr.smithi089.srlyxf) 2024-09-21T04:11:14.993 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:11:14 smithi089 ceph-mon[23733]: Failing over to other MGR 2024-09-21T04:11:14.993 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:11:14 smithi089 ceph-mon[23733]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd='[{"prefix": "mgr fail", "who": "smithi089.srlyxf"}]': finished 2024-09-21T04:11:14.994 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:11:14 smithi089 ceph-mon[23733]: mgrmap e23: smithi195.cgotzm(active, starting, since 0.9976s) 2024-09-21T04:11:14.994 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:11:14 smithi089 ceph-mon[23733]: from='mgr.24407 172.21.15.195:0/751858916' entity='mgr.smithi195.cgotzm' cmd=[{"prefix": "mon metadata", "id": "smithi089"}]: dispatch 2024-09-21T04:11:14.994 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:11:14 smithi089 ceph-mon[23733]: from='mgr.24407 172.21.15.195:0/751858916' entity='mgr.smithi195.cgotzm' cmd=[{"prefix": "mon metadata", "id": "smithi195"}]: dispatch 2024-09-21T04:11:14.994 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:11:14 smithi089 ceph-mon[23733]: from='mgr.24407 172.21.15.195:0/751858916' entity='mgr.smithi195.cgotzm' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi089.osufws"}]: dispatch 2024-09-21T04:11:14.994 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:11:14 smithi089 ceph-mon[23733]: from='mgr.24407 172.21.15.195:0/751858916' entity='mgr.smithi195.cgotzm' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi195.hijxre"}]: dispatch 2024-09-21T04:11:14.994 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:11:14 smithi089 ceph-mon[23733]: from='mgr.24407 172.21.15.195:0/751858916' entity='mgr.smithi195.cgotzm' cmd=[{"prefix": "mgr metadata", "who": "smithi195.cgotzm", "id": "smithi195.cgotzm"}]: dispatch 2024-09-21T04:11:14.994 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:11:14 smithi089 ceph-mon[23733]: from='mgr.24407 172.21.15.195:0/751858916' entity='mgr.smithi195.cgotzm' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-21T04:11:14.994 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:11:14 smithi089 ceph-mon[23733]: from='mgr.24407 172.21.15.195:0/751858916' entity='mgr.smithi195.cgotzm' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-21T04:11:14.994 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:11:14 smithi089 ceph-mon[23733]: from='mgr.24407 172.21.15.195:0/751858916' entity='mgr.smithi195.cgotzm' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-21T04:11:14.994 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:11:14 smithi089 ceph-mon[23733]: from='mgr.24407 172.21.15.195:0/751858916' entity='mgr.smithi195.cgotzm' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-21T04:11:14.994 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:11:14 smithi089 ceph-mon[23733]: from='mgr.24407 172.21.15.195:0/751858916' entity='mgr.smithi195.cgotzm' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-21T04:11:14.994 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:11:14 smithi089 ceph-mon[23733]: from='mgr.24407 172.21.15.195:0/751858916' entity='mgr.smithi195.cgotzm' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-21T04:11:14.994 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:11:14 smithi089 ceph-mon[23733]: from='mgr.24407 172.21.15.195:0/751858916' entity='mgr.smithi195.cgotzm' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-21T04:11:14.994 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:11:14 smithi089 ceph-mon[23733]: from='mgr.24407 172.21.15.195:0/751858916' entity='mgr.smithi195.cgotzm' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-21T04:11:14.995 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:11:14 smithi089 ceph-mon[23733]: from='mgr.24407 172.21.15.195:0/751858916' entity='mgr.smithi195.cgotzm' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-21T04:11:14.995 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:11:14 smithi089 ceph-mon[23733]: from='mgr.24407 172.21.15.195:0/751858916' entity='mgr.smithi195.cgotzm' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-21T04:11:14.995 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:11:14 smithi089 ceph-mon[23733]: from='mgr.24407 172.21.15.195:0/751858916' entity='mgr.smithi195.cgotzm' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-21T04:11:14.995 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:11:14 smithi089 ceph-mon[23733]: Manager daemon smithi195.cgotzm is now available 2024-09-21T04:11:15.308 INFO:teuthology.orchestra.run.smithi089.stdout: 5 62170 15.41 MB/sec warmup 114 sec latency 64.845 ms 2024-09-21T04:11:15.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:11:14 smithi195 ceph-mon[25840]: pgmap v298: 97 pgs: 97 active+clean; 323 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 10 MiB/s wr, 1.00k op/s 2024-09-21T04:11:15.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:11:14 smithi195 ceph-mon[25840]: Upgrade: Need to upgrade myself (mgr.smithi089.srlyxf) 2024-09-21T04:11:15.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:11:14 smithi195 ceph-mon[25840]: Failing over to other MGR 2024-09-21T04:11:15.360 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:11:14 smithi195 ceph-mon[25840]: from='mgr.14200 172.21.15.89:0/3160198909' entity='mgr.smithi089.srlyxf' cmd='[{"prefix": "mgr fail", "who": "smithi089.srlyxf"}]': finished 2024-09-21T04:11:15.360 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:11:14 smithi195 ceph-mon[25840]: mgrmap e23: smithi195.cgotzm(active, starting, since 0.9976s) 2024-09-21T04:11:15.360 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:11:14 smithi195 ceph-mon[25840]: from='mgr.24407 172.21.15.195:0/751858916' entity='mgr.smithi195.cgotzm' cmd=[{"prefix": "mon metadata", "id": "smithi089"}]: dispatch 2024-09-21T04:11:15.360 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:11:14 smithi195 ceph-mon[25840]: from='mgr.24407 172.21.15.195:0/751858916' entity='mgr.smithi195.cgotzm' cmd=[{"prefix": "mon metadata", "id": "smithi195"}]: dispatch 2024-09-21T04:11:15.360 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:11:14 smithi195 ceph-mon[25840]: from='mgr.24407 172.21.15.195:0/751858916' entity='mgr.smithi195.cgotzm' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi089.osufws"}]: dispatch 2024-09-21T04:11:15.360 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:11:14 smithi195 ceph-mon[25840]: from='mgr.24407 172.21.15.195:0/751858916' entity='mgr.smithi195.cgotzm' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi195.hijxre"}]: dispatch 2024-09-21T04:11:15.360 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:11:14 smithi195 ceph-mon[25840]: from='mgr.24407 172.21.15.195:0/751858916' entity='mgr.smithi195.cgotzm' cmd=[{"prefix": "mgr metadata", "who": "smithi195.cgotzm", "id": "smithi195.cgotzm"}]: dispatch 2024-09-21T04:11:15.361 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:11:14 smithi195 ceph-mon[25840]: from='mgr.24407 172.21.15.195:0/751858916' entity='mgr.smithi195.cgotzm' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-21T04:11:15.361 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:11:14 smithi195 ceph-mon[25840]: from='mgr.24407 172.21.15.195:0/751858916' entity='mgr.smithi195.cgotzm' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-21T04:11:15.361 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:11:14 smithi195 ceph-mon[25840]: from='mgr.24407 172.21.15.195:0/751858916' entity='mgr.smithi195.cgotzm' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-21T04:11:15.361 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:11:14 smithi195 ceph-mon[25840]: from='mgr.24407 172.21.15.195:0/751858916' entity='mgr.smithi195.cgotzm' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-21T04:11:15.361 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:11:14 smithi195 ceph-mon[25840]: from='mgr.24407 172.21.15.195:0/751858916' entity='mgr.smithi195.cgotzm' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-21T04:11:15.361 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:11:14 smithi195 ceph-mon[25840]: from='mgr.24407 172.21.15.195:0/751858916' entity='mgr.smithi195.cgotzm' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-21T04:11:15.361 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:11:14 smithi195 ceph-mon[25840]: from='mgr.24407 172.21.15.195:0/751858916' entity='mgr.smithi195.cgotzm' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-21T04:11:15.361 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:11:14 smithi195 ceph-mon[25840]: from='mgr.24407 172.21.15.195:0/751858916' entity='mgr.smithi195.cgotzm' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-21T04:11:15.361 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:11:14 smithi195 ceph-mon[25840]: from='mgr.24407 172.21.15.195:0/751858916' entity='mgr.smithi195.cgotzm' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-21T04:11:15.361 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:11:14 smithi195 ceph-mon[25840]: from='mgr.24407 172.21.15.195:0/751858916' entity='mgr.smithi195.cgotzm' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-21T04:11:15.362 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:11:14 smithi195 ceph-mon[25840]: from='mgr.24407 172.21.15.195:0/751858916' entity='mgr.smithi195.cgotzm' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-21T04:11:15.362 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:11:14 smithi195 ceph-mon[25840]: Manager daemon smithi195.cgotzm is now available 2024-09-21T04:11:16.308 INFO:teuthology.orchestra.run.smithi089.stdout: 5 62659 15.34 MB/sec warmup 115 sec latency 40.285 ms 2024-09-21T04:11:16.504 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:11:16 smithi089 ceph-mon[23733]: mgrmap e24: smithi195.cgotzm(active, since 2s) 2024-09-21T04:11:16.504 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:11:16 smithi089 ceph-mon[23733]: from='mgr.24407 ' entity='mgr.smithi195.cgotzm' 2024-09-21T04:11:16.504 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:11:16 smithi089 ceph-mon[23733]: from='mgr.24407 ' entity='mgr.smithi195.cgotzm' 2024-09-21T04:11:16.504 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:11:16 smithi089 ceph-mon[23733]: from='mgr.24407 ' entity='mgr.smithi195.cgotzm' 2024-09-21T04:11:16.504 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:11:16 smithi089 ceph-mon[23733]: Checking for cert/key for grafana.smithi089 2024-09-21T04:11:16.504 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:11:16 smithi089 ceph-mon[23733]: from='mgr.24407 ' entity='mgr.smithi195.cgotzm' 2024-09-21T04:11:16.504 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:11:16 smithi089 ceph-mon[23733]: from='mgr.24407 172.21.15.195:0/751858916' entity='mgr.smithi195.cgotzm' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T04:11:16.504 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:11:16 smithi089 ceph-mon[23733]: from='mgr.24407 172.21.15.195:0/751858916' entity='mgr.smithi195.cgotzm' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T04:11:16.504 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:11:16 smithi089 ceph-mon[23733]: from='mgr.24407 172.21.15.195:0/751858916' entity='mgr.smithi195.cgotzm' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-21T04:11:16.504 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:11:16 smithi089 ceph-mon[23733]: from='mon.? -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-21T04:11:16.505 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:11:16 smithi089 ceph-mon[23733]: from='mgr.24407 172.21.15.195:0/751858916' entity='mgr.smithi195.cgotzm' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi195.cgotzm/mirror_snapshot_schedule"}]: dispatch 2024-09-21T04:11:16.505 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:11:16 smithi089 ceph-mon[23733]: from='mgr.24407 ' entity='mgr.smithi195.cgotzm' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi195.cgotzm/mirror_snapshot_schedule"}]: dispatch 2024-09-21T04:11:16.505 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:11:16 smithi089 ceph-mon[23733]: from='mgr.24407 172.21.15.195:0/751858916' entity='mgr.smithi195.cgotzm' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi195.cgotzm/trash_purge_schedule"}]: dispatch 2024-09-21T04:11:16.505 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:11:16 smithi089 ceph-mon[23733]: from='mgr.24407 ' entity='mgr.smithi195.cgotzm' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi195.cgotzm/trash_purge_schedule"}]: dispatch 2024-09-21T04:11:16.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:11:16 smithi195 ceph-mon[25840]: mgrmap e24: smithi195.cgotzm(active, since 2s) 2024-09-21T04:11:16.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:11:16 smithi195 ceph-mon[25840]: from='mgr.24407 ' entity='mgr.smithi195.cgotzm' 2024-09-21T04:11:16.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:11:16 smithi195 ceph-mon[25840]: from='mgr.24407 ' entity='mgr.smithi195.cgotzm' 2024-09-21T04:11:16.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:11:16 smithi195 ceph-mon[25840]: from='mgr.24407 ' entity='mgr.smithi195.cgotzm' 2024-09-21T04:11:16.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:11:16 smithi195 ceph-mon[25840]: Checking for cert/key for grafana.smithi089 2024-09-21T04:11:16.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:11:16 smithi195 ceph-mon[25840]: from='mgr.24407 ' entity='mgr.smithi195.cgotzm' 2024-09-21T04:11:16.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:11:16 smithi195 ceph-mon[25840]: from='mgr.24407 172.21.15.195:0/751858916' entity='mgr.smithi195.cgotzm' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T04:11:16.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:11:16 smithi195 ceph-mon[25840]: from='mgr.24407 172.21.15.195:0/751858916' entity='mgr.smithi195.cgotzm' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T04:11:16.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:11:16 smithi195 ceph-mon[25840]: from='mgr.24407 172.21.15.195:0/751858916' entity='mgr.smithi195.cgotzm' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-21T04:11:16.610 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:11:16 smithi195 ceph-mon[25840]: from='mon.? -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-21T04:11:16.610 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:11:16 smithi195 ceph-mon[25840]: from='mgr.24407 172.21.15.195:0/751858916' entity='mgr.smithi195.cgotzm' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi195.cgotzm/mirror_snapshot_schedule"}]: dispatch 2024-09-21T04:11:16.610 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:11:16 smithi195 ceph-mon[25840]: from='mgr.24407 ' entity='mgr.smithi195.cgotzm' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi195.cgotzm/mirror_snapshot_schedule"}]: dispatch 2024-09-21T04:11:16.611 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:11:16 smithi195 ceph-mon[25840]: from='mgr.24407 172.21.15.195:0/751858916' entity='mgr.smithi195.cgotzm' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi195.cgotzm/trash_purge_schedule"}]: dispatch 2024-09-21T04:11:16.611 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:11:16 smithi195 ceph-mon[25840]: from='mgr.24407 ' entity='mgr.smithi195.cgotzm' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi195.cgotzm/trash_purge_schedule"}]: dispatch 2024-09-21T04:11:17.308 INFO:teuthology.orchestra.run.smithi089.stdout: 5 63125 15.38 MB/sec warmup 116 sec latency 118.616 ms 2024-09-21T04:11:17.519 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:11:17 smithi089 ceph-mon[23733]: pgmap v4: 97 pgs: 97 active+clean; 317 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail 2024-09-21T04:11:17.519 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:11:17 smithi089 ceph-mon[23733]: Deploying cephadm binary to smithi195 2024-09-21T04:11:17.519 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:11:17 smithi089 ceph-mon[23733]: Deploying cephadm binary to smithi089 2024-09-21T04:11:17.608 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:11:17 smithi195 ceph-mon[25840]: pgmap v4: 97 pgs: 97 active+clean; 317 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail 2024-09-21T04:11:17.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:11:17 smithi195 ceph-mon[25840]: Deploying cephadm binary to smithi195 2024-09-21T04:11:17.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:11:17 smithi195 ceph-mon[25840]: Deploying cephadm binary to smithi089 2024-09-21T04:11:18.308 INFO:teuthology.orchestra.run.smithi089.stdout: 5 63748 15.45 MB/sec warmup 117 sec latency 39.663 ms 2024-09-21T04:11:18.533 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:11:18 smithi089 ceph-mon[23733]: mgrmap e25: smithi195.cgotzm(active, since 4s) 2024-09-21T04:11:18.533 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:11:18 smithi089 ceph-mon[23733]: [21/Sep/2024:04:11:17] ENGINE Bus STARTING 2024-09-21T04:11:18.533 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:11:18 smithi089 ceph-mon[23733]: [21/Sep/2024:04:11:17] ENGINE Serving on https://172.21.15.195:7150 2024-09-21T04:11:18.533 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:11:18 smithi089 ceph-mon[23733]: [21/Sep/2024:04:11:17] ENGINE Error in HTTPServer.serve 2024-09-21T04:11:18.533 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout: Traceback (most recent call last): 2024-09-21T04:11:18.533 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-21T04:11:18.533 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout: self._connections.run(self.expiration_interval) 2024-09-21T04:11:18.534 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-21T04:11:18.534 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout: self._run(expiration_interval) 2024-09-21T04:11:18.534 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-21T04:11:18.534 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-21T04:11:18.534 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-21T04:11:18.534 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-21T04:11:18.534 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-21T04:11:18.534 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout: s = self.context.wrap_socket( 2024-09-21T04:11:18.535 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-21T04:11:18.535 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout: return self.sslsocket_class._create( 2024-09-21T04:11:18.535 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-21T04:11:18.535 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout: self.do_handshake() 2024-09-21T04:11:18.535 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-21T04:11:18.535 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout: self._sslobj.do_handshake() 2024-09-21T04:11:18.535 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-21T04:11:18.535 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:11:18 smithi089 ceph-mon[23733]: [21/Sep/2024:04:11:17] ENGINE Serving on http://172.21.15.195:8765 2024-09-21T04:11:18.535 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:11:18 smithi089 ceph-mon[23733]: [21/Sep/2024:04:11:17] ENGINE Bus STARTED 2024-09-21T04:11:18.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:11:18 smithi195 ceph-mon[25840]: mgrmap e25: smithi195.cgotzm(active, since 4s) 2024-09-21T04:11:18.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:11:18 smithi195 ceph-mon[25840]: [21/Sep/2024:04:11:17] ENGINE Bus STARTING 2024-09-21T04:11:18.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:11:18 smithi195 ceph-mon[25840]: [21/Sep/2024:04:11:17] ENGINE Serving on https://172.21.15.195:7150 2024-09-21T04:11:18.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:11:18 smithi195 ceph-mon[25840]: [21/Sep/2024:04:11:17] ENGINE Error in HTTPServer.serve 2024-09-21T04:11:18.610 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout: Traceback (most recent call last): 2024-09-21T04:11:18.610 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-21T04:11:18.610 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout: self._connections.run(self.expiration_interval) 2024-09-21T04:11:18.610 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-21T04:11:18.610 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout: self._run(expiration_interval) 2024-09-21T04:11:18.610 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-21T04:11:18.611 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-21T04:11:18.611 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-21T04:11:18.611 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-21T04:11:18.611 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-21T04:11:18.611 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout: s = self.context.wrap_socket( 2024-09-21T04:11:18.611 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-21T04:11:18.612 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout: return self.sslsocket_class._create( 2024-09-21T04:11:18.612 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-21T04:11:18.612 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout: self.do_handshake() 2024-09-21T04:11:18.612 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-21T04:11:18.612 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout: self._sslobj.do_handshake() 2024-09-21T04:11:18.612 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-21T04:11:18.613 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:11:18 smithi195 ceph-mon[25840]: [21/Sep/2024:04:11:17] ENGINE Serving on http://172.21.15.195:8765 2024-09-21T04:11:18.613 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:11:18 smithi195 ceph-mon[25840]: [21/Sep/2024:04:11:17] ENGINE Bus STARTED 2024-09-21T04:11:19.308 INFO:teuthology.orchestra.run.smithi089.stdout: 5 64113 15.45 MB/sec warmup 118 sec latency 38.501 ms 2024-09-21T04:11:19.547 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:11:19 smithi089 ceph-mon[23733]: pgmap v5: 97 pgs: 97 active+clean; 317 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail 2024-09-21T04:11:19.859 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:11:19 smithi195 ceph-mon[25840]: pgmap v5: 97 pgs: 97 active+clean; 317 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail 2024-09-21T04:11:20.309 INFO:teuthology.orchestra.run.smithi089.stdout: 5 64401 15.34 MB/sec warmup 119 sec latency 44.155 ms 2024-09-21T04:11:20.558 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:11:20 smithi089 ceph-mon[23733]: mgrmap e26: smithi195.cgotzm(active, since 6s) 2024-09-21T04:11:20.824 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:11:20 smithi195 ceph-mon[25840]: mgrmap e26: smithi195.cgotzm(active, since 6s) 2024-09-21T04:11:21.565 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:11:21 smithi089 ceph-mon[23733]: pgmap v6: 97 pgs: 97 active+clean; 318 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 5.8 KiB/s rd, 13 KiB/s wr, 12 op/s 2024-09-21T04:11:21.770 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:11:21 smithi195 ceph-mon[25840]: pgmap v6: 97 pgs: 97 active+clean; 318 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 5.8 KiB/s rd, 13 KiB/s wr, 12 op/s 2024-09-21T04:11:22.311 INFO:teuthology.orchestra.run.smithi089.stdout: 5 65233 14.52 MB/sec execute 1 sec latency 30.765 ms 2024-09-21T04:11:23.314 INFO:teuthology.orchestra.run.smithi089.stdout: 5 65767 11.17 MB/sec execute 2 sec latency 59.278 ms 2024-09-21T04:11:23.342 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:11:23 smithi089 ceph-mon[23733]: from='mgr.24407 ' entity='mgr.smithi195.cgotzm' 2024-09-21T04:11:23.342 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:11:23 smithi089 ceph-mon[23733]: from='mgr.24407 ' entity='mgr.smithi195.cgotzm' 2024-09-21T04:11:23.342 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:11:23 smithi089 ceph-mon[23733]: pgmap v7: 97 pgs: 97 active+clean; 334 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 29 KiB/s rd, 5.5 MiB/s wr, 574 op/s 2024-09-21T04:11:23.342 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:11:23 smithi089 ceph-mon[23733]: from='mgr.24407 ' entity='mgr.smithi195.cgotzm' 2024-09-21T04:11:23.342 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:11:23 smithi089 ceph-mon[23733]: from='mgr.24407 ' entity='mgr.smithi195.cgotzm' 2024-09-21T04:11:23.569 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:11:23 smithi195 ceph-mon[25840]: from='mgr.24407 ' entity='mgr.smithi195.cgotzm' 2024-09-21T04:11:23.569 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:11:23 smithi195 ceph-mon[25840]: from='mgr.24407 ' entity='mgr.smithi195.cgotzm' 2024-09-21T04:11:23.569 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:11:23 smithi195 ceph-mon[25840]: pgmap v7: 97 pgs: 97 active+clean; 334 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 29 KiB/s rd, 5.5 MiB/s wr, 574 op/s 2024-09-21T04:11:23.569 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:11:23 smithi195 ceph-mon[25840]: from='mgr.24407 ' entity='mgr.smithi195.cgotzm' 2024-09-21T04:11:23.570 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:11:23 smithi195 ceph-mon[25840]: from='mgr.24407 ' entity='mgr.smithi195.cgotzm' 2024-09-21T04:11:24.314 INFO:teuthology.orchestra.run.smithi089.stdout: 5 66333 10.69 MB/sec execute 3 sec latency 33.772 ms 2024-09-21T04:11:24.453 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:11:24 smithi089 ceph-mon[23733]: Standby manager daemon smithi089.srlyxf started 2024-09-21T04:11:24.453 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:11:24 smithi089 ceph-mon[23733]: from='mgr.? 172.21.15.89:0/192958679' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi089.srlyxf/crt"}]: dispatch 2024-09-21T04:11:24.453 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:11:24 smithi089 ceph-mon[23733]: from='mgr.? 172.21.15.89:0/192958679' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-21T04:11:24.453 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:11:24 smithi089 ceph-mon[23733]: from='mgr.? 172.21.15.89:0/192958679' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi089.srlyxf/key"}]: dispatch 2024-09-21T04:11:24.453 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:11:24 smithi089 ceph-mon[23733]: from='mgr.? 172.21.15.89:0/192958679' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-21T04:11:24.608 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:11:24 smithi195 ceph-mon[25840]: Standby manager daemon smithi089.srlyxf started 2024-09-21T04:11:24.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:11:24 smithi195 ceph-mon[25840]: from='mgr.? 172.21.15.89:0/192958679' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi089.srlyxf/crt"}]: dispatch 2024-09-21T04:11:24.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:11:24 smithi195 ceph-mon[25840]: from='mgr.? 172.21.15.89:0/192958679' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-21T04:11:24.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:11:24 smithi195 ceph-mon[25840]: from='mgr.? 172.21.15.89:0/192958679' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi089.srlyxf/key"}]: dispatch 2024-09-21T04:11:24.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:11:24 smithi195 ceph-mon[25840]: from='mgr.? 172.21.15.89:0/192958679' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-21T04:11:25.314 INFO:teuthology.orchestra.run.smithi089.stdout: 5 66935 14.78 MB/sec execute 4 sec latency 31.096 ms 2024-09-21T04:11:25.467 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:11:25 smithi089 ceph-mon[23733]: mgrmap e27: smithi195.cgotzm(active, since 10s), standbys: smithi089.srlyxf 2024-09-21T04:11:25.468 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:11:25 smithi089 ceph-mon[23733]: from='mgr.24407 172.21.15.195:0/751858916' entity='mgr.smithi195.cgotzm' cmd=[{"prefix": "mgr metadata", "who": "smithi089.srlyxf", "id": "smithi089.srlyxf"}]: dispatch 2024-09-21T04:11:25.468 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:11:25 smithi089 ceph-mon[23733]: pgmap v8: 97 pgs: 97 active+clean; 334 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 23 KiB/s rd, 4.3 MiB/s wr, 446 op/s 2024-09-21T04:11:25.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:11:25 smithi195 ceph-mon[25840]: mgrmap e27: smithi195.cgotzm(active, since 10s), standbys: smithi089.srlyxf 2024-09-21T04:11:25.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:11:25 smithi195 ceph-mon[25840]: from='mgr.24407 172.21.15.195:0/751858916' entity='mgr.smithi195.cgotzm' cmd=[{"prefix": "mgr metadata", "who": "smithi089.srlyxf", "id": "smithi089.srlyxf"}]: dispatch 2024-09-21T04:11:25.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:11:25 smithi195 ceph-mon[25840]: pgmap v8: 97 pgs: 97 active+clean; 334 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 23 KiB/s rd, 4.3 MiB/s wr, 446 op/s 2024-09-21T04:11:26.314 INFO:teuthology.orchestra.run.smithi089.stdout: 5 67440 16.19 MB/sec execute 5 sec latency 28.892 ms 2024-09-21T04:11:26.566 INFO:teuthology.orchestra.run.smithi089.stdout:true 2024-09-21T04:11:27.314 INFO:teuthology.orchestra.run.smithi089.stdout: 5 67775 15.00 MB/sec execute 6 sec latency 42.772 ms 2024-09-21T04:11:27.582 INFO:teuthology.orchestra.run.smithi089.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-21T04:11:27.582 INFO:teuthology.orchestra.run.smithi089.stdout:alertmanager.smithi089 smithi089 *:9093,9094 running (7m) 0s ago 8m 26.2M - 0.23.0 ba2b418f427c 703c928c9523 2024-09-21T04:11:27.582 INFO:teuthology.orchestra.run.smithi089.stdout:crash.smithi089 smithi089 running (8m) 0s ago 8m 6409k - 17.2.0 e1d6a67b021e d1e0a8969f78 2024-09-21T04:11:27.582 INFO:teuthology.orchestra.run.smithi089.stdout:crash.smithi195 smithi195 running (8m) 5s ago 8m 6400k - 17.2.0 e1d6a67b021e 7e213ff76084 2024-09-21T04:11:27.582 INFO:teuthology.orchestra.run.smithi089.stdout:grafana.smithi089 smithi089 *:3000 running (7m) 0s ago 8m 53.4M - dad864ee21e9 547de193a0ee 2024-09-21T04:11:27.582 INFO:teuthology.orchestra.run.smithi089.stdout:mds.foofs.smithi089.osufws smithi089 running (2m) 0s ago 2m 43.4M - 17.2.0 e1d6a67b021e 664d41859bea 2024-09-21T04:11:27.582 INFO:teuthology.orchestra.run.smithi089.stdout:mds.foofs.smithi195.hijxre smithi195 running (2m) 5s ago 2m 14.9M - 17.2.0 e1d6a67b021e 7bc0c6542d12 2024-09-21T04:11:27.582 INFO:teuthology.orchestra.run.smithi089.stdout:mgr.smithi089.srlyxf smithi089 *:9283 running (9m) 0s ago 9m 265M - 17.2.0 e1d6a67b021e 410bb00e916d 2024-09-21T04:11:27.582 INFO:teuthology.orchestra.run.smithi089.stdout:mgr.smithi195.cgotzm smithi195 *:8443,9283 running (35s) 5s ago 7m 499M - 19.3.0-5103-gf36283e7 48ed0a2cf89d 18c1e9ebdeda 2024-09-21T04:11:27.582 INFO:teuthology.orchestra.run.smithi089.stdout:mon.smithi089 smithi089 running (9m) 0s ago 9m 51.7M 2048M 17.2.0 e1d6a67b021e 1d2b3719e46e 2024-09-21T04:11:27.582 INFO:teuthology.orchestra.run.smithi089.stdout:mon.smithi195 smithi195 running (7m) 5s ago 7m 40.2M 2048M 17.2.0 e1d6a67b021e 7f86c5b5d21e 2024-09-21T04:11:27.582 INFO:teuthology.orchestra.run.smithi089.stdout:nfs.foo.0.0.smithi089.dcjmqc smithi089 *:2049 running (2m) 0s ago 2m 243M - 4.0 e1d6a67b021e 98550f94e773 2024-09-21T04:11:27.583 INFO:teuthology.orchestra.run.smithi089.stdout:nfs.foo.1.0.smithi195.vhoayp smithi195 *:2049 running (2m) 5s ago 2m 75.5M - 4.0 e1d6a67b021e c60a4040130a 2024-09-21T04:11:27.583 INFO:teuthology.orchestra.run.smithi089.stdout:node-exporter.smithi089 smithi089 *:9100 running (8m) 0s ago 8m 21.0M - 1.3.1 1dbe0e931976 5b166976fc1c 2024-09-21T04:11:27.583 INFO:teuthology.orchestra.run.smithi089.stdout:node-exporter.smithi195 smithi195 *:9100 running (7m) 5s ago 7m 18.4M - 1.3.1 1dbe0e931976 3448624c239b 2024-09-21T04:11:27.583 INFO:teuthology.orchestra.run.smithi089.stdout:osd.0 smithi089 running (7m) 0s ago 7m 283M 3215M 17.2.0 e1d6a67b021e db7c0e5020e7 2024-09-21T04:11:27.583 INFO:teuthology.orchestra.run.smithi089.stdout:osd.1 smithi089 running (6m) 0s ago 6m 206M 3215M 17.2.0 e1d6a67b021e f9ec697d08d0 2024-09-21T04:11:27.583 INFO:teuthology.orchestra.run.smithi089.stdout:osd.2 smithi089 running (6m) 0s ago 6m 191M 3215M 17.2.0 e1d6a67b021e cfa000948859 2024-09-21T04:11:27.583 INFO:teuthology.orchestra.run.smithi089.stdout:osd.3 smithi089 running (5m) 0s ago 5m 225M 3215M 17.2.0 e1d6a67b021e 31c610ed5815 2024-09-21T04:11:27.583 INFO:teuthology.orchestra.run.smithi089.stdout:osd.4 smithi195 running (5m) 5s ago 5m 246M 2704M 17.2.0 e1d6a67b021e f00e18a1f220 2024-09-21T04:11:27.583 INFO:teuthology.orchestra.run.smithi089.stdout:osd.5 smithi195 running (5m) 5s ago 5m 278M 2704M 17.2.0 e1d6a67b021e cecfd6ff1eee 2024-09-21T04:11:27.583 INFO:teuthology.orchestra.run.smithi089.stdout:osd.6 smithi195 running (4m) 5s ago 4m 246M 2704M 17.2.0 e1d6a67b021e 0d7db362cc23 2024-09-21T04:11:27.583 INFO:teuthology.orchestra.run.smithi089.stdout:osd.7 smithi195 running (4m) 5s ago 4m 328M 2704M 17.2.0 e1d6a67b021e 0109b3fea188 2024-09-21T04:11:27.583 INFO:teuthology.orchestra.run.smithi089.stdout:prometheus.smithi089 smithi089 *:9095 running (7m) 0s ago 7m 65.0M - 2.33.4 514e6a882f6e 9ed6ce8f2de3 2024-09-21T04:11:27.737 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:11:27 smithi089 ceph-mon[23733]: pgmap v9: 97 pgs: 97 active+clean; 331 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 24 KiB/s rd, 6.6 MiB/s wr, 698 op/s 2024-09-21T04:11:27.737 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:11:27 smithi089 ceph-mon[23733]: from='client.14728 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T04:11:27.737 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:11:27 smithi089 ceph-mon[23733]: from='mgr.24407 ' entity='mgr.smithi195.cgotzm' 2024-09-21T04:11:27.737 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:11:27 smithi089 ceph-mon[23733]: from='mgr.24407 ' entity='mgr.smithi195.cgotzm' 2024-09-21T04:11:27.737 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:11:27 smithi089 ceph-mon[23733]: from='mgr.24407 172.21.15.195:0/751858916' entity='mgr.smithi195.cgotzm' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-21T04:11:27.737 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:11:27 smithi089 ceph-mon[23733]: from='mgr.24407 ' entity='mgr.smithi195.cgotzm' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-21T04:11:27.737 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:11:27 smithi089 ceph-mon[23733]: from='mgr.24407 172.21.15.195:0/751858916' entity='mgr.smithi195.cgotzm' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-21T04:11:27.737 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:11:27 smithi089 ceph-mon[23733]: from='mgr.24407 ' entity='mgr.smithi195.cgotzm' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-21T04:11:27.738 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:11:27 smithi089 ceph-mon[23733]: from='mgr.24407 172.21.15.195:0/751858916' entity='mgr.smithi195.cgotzm' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-21T04:11:27.738 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:11:27 smithi089 ceph-mon[23733]: from='mgr.24407 ' entity='mgr.smithi195.cgotzm' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-21T04:11:27.738 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:11:27 smithi089 ceph-mon[23733]: from='mgr.24407 172.21.15.195:0/751858916' entity='mgr.smithi195.cgotzm' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-21T04:11:27.738 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:11:27 smithi089 ceph-mon[23733]: from='mgr.24407 ' entity='mgr.smithi195.cgotzm' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-21T04:11:27.738 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:11:27 smithi089 ceph-mon[23733]: Adjusting osd_memory_target on smithi195 to 2704M 2024-09-21T04:11:27.738 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:11:27 smithi089 ceph-mon[23733]: from='mgr.24407 ' entity='mgr.smithi195.cgotzm' 2024-09-21T04:11:27.738 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:11:27 smithi089 ceph-mon[23733]: from='client.14732 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T04:11:27.738 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:11:27 smithi089 ceph-mon[23733]: from='mgr.24407 ' entity='mgr.smithi195.cgotzm' 2024-09-21T04:11:27.738 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:11:27 smithi089 ceph-mon[23733]: from='mgr.24407 ' entity='mgr.smithi195.cgotzm' 2024-09-21T04:11:27.859 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:11:27 smithi195 ceph-mon[25840]: pgmap v9: 97 pgs: 97 active+clean; 331 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 24 KiB/s rd, 6.6 MiB/s wr, 698 op/s 2024-09-21T04:11:27.859 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:11:27 smithi195 ceph-mon[25840]: from='client.14728 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T04:11:27.859 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:11:27 smithi195 ceph-mon[25840]: from='mgr.24407 ' entity='mgr.smithi195.cgotzm' 2024-09-21T04:11:27.859 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:11:27 smithi195 ceph-mon[25840]: from='mgr.24407 ' entity='mgr.smithi195.cgotzm' 2024-09-21T04:11:27.859 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:11:27 smithi195 ceph-mon[25840]: from='mgr.24407 172.21.15.195:0/751858916' entity='mgr.smithi195.cgotzm' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-21T04:11:27.859 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:11:27 smithi195 ceph-mon[25840]: from='mgr.24407 ' entity='mgr.smithi195.cgotzm' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-21T04:11:27.859 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:11:27 smithi195 ceph-mon[25840]: from='mgr.24407 172.21.15.195:0/751858916' entity='mgr.smithi195.cgotzm' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-21T04:11:27.859 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:11:27 smithi195 ceph-mon[25840]: from='mgr.24407 ' entity='mgr.smithi195.cgotzm' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-21T04:11:27.859 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:11:27 smithi195 ceph-mon[25840]: from='mgr.24407 172.21.15.195:0/751858916' entity='mgr.smithi195.cgotzm' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-21T04:11:27.860 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:11:27 smithi195 ceph-mon[25840]: from='mgr.24407 ' entity='mgr.smithi195.cgotzm' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-21T04:11:27.860 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:11:27 smithi195 ceph-mon[25840]: from='mgr.24407 172.21.15.195:0/751858916' entity='mgr.smithi195.cgotzm' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-21T04:11:27.860 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:11:27 smithi195 ceph-mon[25840]: from='mgr.24407 ' entity='mgr.smithi195.cgotzm' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-21T04:11:27.860 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:11:27 smithi195 ceph-mon[25840]: Adjusting osd_memory_target on smithi195 to 2704M 2024-09-21T04:11:27.860 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:11:27 smithi195 ceph-mon[25840]: from='mgr.24407 ' entity='mgr.smithi195.cgotzm' 2024-09-21T04:11:27.860 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:11:27 smithi195 ceph-mon[25840]: from='client.14732 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T04:11:27.860 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:11:27 smithi195 ceph-mon[25840]: from='mgr.24407 ' entity='mgr.smithi195.cgotzm' 2024-09-21T04:11:27.860 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:11:27 smithi195 ceph-mon[25840]: from='mgr.24407 ' entity='mgr.smithi195.cgotzm' 2024-09-21T04:11:28.307 INFO:teuthology.orchestra.run.smithi089.stdout:{ 2024-09-21T04:11:28.308 INFO:teuthology.orchestra.run.smithi089.stdout: "mon": { 2024-09-21T04:11:28.308 INFO:teuthology.orchestra.run.smithi089.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-21T04:11:28.308 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2024-09-21T04:11:28.308 INFO:teuthology.orchestra.run.smithi089.stdout: "mgr": { 2024-09-21T04:11:28.308 INFO:teuthology.orchestra.run.smithi089.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 1, 2024-09-21T04:11:28.308 INFO:teuthology.orchestra.run.smithi089.stdout: "ceph version 19.3.0-5103-gf36283e7 (f36283e7c91cac1c01cb9a7c4ff21bd697dbd4f2) squid (dev)": 1 2024-09-21T04:11:28.308 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2024-09-21T04:11:28.308 INFO:teuthology.orchestra.run.smithi089.stdout: "osd": { 2024-09-21T04:11:28.308 INFO:teuthology.orchestra.run.smithi089.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-09-21T04:11:28.308 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2024-09-21T04:11:28.308 INFO:teuthology.orchestra.run.smithi089.stdout: "mds": { 2024-09-21T04:11:28.308 INFO:teuthology.orchestra.run.smithi089.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-21T04:11:28.308 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2024-09-21T04:11:28.308 INFO:teuthology.orchestra.run.smithi089.stdout: "overall": { 2024-09-21T04:11:28.308 INFO:teuthology.orchestra.run.smithi089.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 13, 2024-09-21T04:11:28.309 INFO:teuthology.orchestra.run.smithi089.stdout: "ceph version 19.3.0-5103-gf36283e7 (f36283e7c91cac1c01cb9a7c4ff21bd697dbd4f2) squid (dev)": 1 2024-09-21T04:11:28.309 INFO:teuthology.orchestra.run.smithi089.stdout: } 2024-09-21T04:11:28.309 INFO:teuthology.orchestra.run.smithi089.stdout:} 2024-09-21T04:11:28.314 INFO:teuthology.orchestra.run.smithi089.stdout: 5 68214 13.27 MB/sec execute 7 sec latency 22.676 ms 2024-09-21T04:11:28.746 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:11:28 smithi089 ceph-mon[23733]: from='client.14736 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T04:11:28.746 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:11:28 smithi089 ceph-mon[23733]: from='mgr.24407 ' entity='mgr.smithi195.cgotzm' 2024-09-21T04:11:28.746 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:11:28 smithi089 ceph-mon[23733]: from='mgr.24407 ' entity='mgr.smithi195.cgotzm' 2024-09-21T04:11:28.746 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:11:28 smithi089 ceph-mon[23733]: from='mgr.24407 172.21.15.195:0/751858916' entity='mgr.smithi195.cgotzm' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-21T04:11:28.746 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:11:28 smithi089 ceph-mon[23733]: from='mgr.24407 ' entity='mgr.smithi195.cgotzm' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-21T04:11:28.746 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:11:28 smithi089 ceph-mon[23733]: from='mgr.24407 172.21.15.195:0/751858916' entity='mgr.smithi195.cgotzm' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-21T04:11:28.747 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:11:28 smithi089 ceph-mon[23733]: from='mgr.24407 ' entity='mgr.smithi195.cgotzm' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-21T04:11:28.747 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:11:28 smithi089 ceph-mon[23733]: from='mgr.24407 172.21.15.195:0/751858916' entity='mgr.smithi195.cgotzm' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-21T04:11:28.747 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:11:28 smithi089 ceph-mon[23733]: from='mgr.24407 ' entity='mgr.smithi195.cgotzm' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-21T04:11:28.747 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:11:28 smithi089 ceph-mon[23733]: from='mgr.24407 172.21.15.195:0/751858916' entity='mgr.smithi195.cgotzm' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-21T04:11:28.747 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:11:28 smithi089 ceph-mon[23733]: from='mgr.24407 ' entity='mgr.smithi195.cgotzm' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-21T04:11:28.747 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:11:28 smithi089 ceph-mon[23733]: from='mgr.24407 ' entity='mgr.smithi195.cgotzm' 2024-09-21T04:11:28.747 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:11:28 smithi089 ceph-mon[23733]: from='mgr.24407 172.21.15.195:0/751858916' entity='mgr.smithi195.cgotzm' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T04:11:28.747 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:11:28 smithi089 ceph-mon[23733]: from='mgr.24407 172.21.15.195:0/751858916' entity='mgr.smithi195.cgotzm' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T04:11:28.747 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:11:28 smithi089 ceph-mon[23733]: from='client.? 172.21.15.89:0/2940062081' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T04:11:28.859 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:11:28 smithi195 ceph-mon[25840]: from='client.14736 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T04:11:28.859 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:11:28 smithi195 ceph-mon[25840]: from='mgr.24407 ' entity='mgr.smithi195.cgotzm' 2024-09-21T04:11:28.859 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:11:28 smithi195 ceph-mon[25840]: from='mgr.24407 ' entity='mgr.smithi195.cgotzm' 2024-09-21T04:11:28.859 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:11:28 smithi195 ceph-mon[25840]: from='mgr.24407 172.21.15.195:0/751858916' entity='mgr.smithi195.cgotzm' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-21T04:11:28.859 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:11:28 smithi195 ceph-mon[25840]: from='mgr.24407 ' entity='mgr.smithi195.cgotzm' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-21T04:11:28.859 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:11:28 smithi195 ceph-mon[25840]: from='mgr.24407 172.21.15.195:0/751858916' entity='mgr.smithi195.cgotzm' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-21T04:11:28.859 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:11:28 smithi195 ceph-mon[25840]: from='mgr.24407 ' entity='mgr.smithi195.cgotzm' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-21T04:11:28.859 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:11:28 smithi195 ceph-mon[25840]: from='mgr.24407 172.21.15.195:0/751858916' entity='mgr.smithi195.cgotzm' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-21T04:11:28.859 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:11:28 smithi195 ceph-mon[25840]: from='mgr.24407 ' entity='mgr.smithi195.cgotzm' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-21T04:11:28.859 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:11:28 smithi195 ceph-mon[25840]: from='mgr.24407 172.21.15.195:0/751858916' entity='mgr.smithi195.cgotzm' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-21T04:11:28.859 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:11:28 smithi195 ceph-mon[25840]: from='mgr.24407 ' entity='mgr.smithi195.cgotzm' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-21T04:11:28.860 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:11:28 smithi195 ceph-mon[25840]: from='mgr.24407 ' entity='mgr.smithi195.cgotzm' 2024-09-21T04:11:28.860 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:11:28 smithi195 ceph-mon[25840]: from='mgr.24407 172.21.15.195:0/751858916' entity='mgr.smithi195.cgotzm' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T04:11:28.860 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:11:28 smithi195 ceph-mon[25840]: from='mgr.24407 172.21.15.195:0/751858916' entity='mgr.smithi195.cgotzm' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T04:11:28.860 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:11:28 smithi195 ceph-mon[25840]: from='client.? 172.21.15.89:0/2940062081' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T04:11:28.944 INFO:teuthology.orchestra.run.smithi089.stdout:{ 2024-09-21T04:11:28.944 INFO:teuthology.orchestra.run.smithi089.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:2195e5f178be0a9390728e2a14088236d8fe6f82635f5c84418fe37a04457e00", 2024-09-21T04:11:28.944 INFO:teuthology.orchestra.run.smithi089.stdout: "in_progress": true, 2024-09-21T04:11:28.944 INFO:teuthology.orchestra.run.smithi089.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-21T04:11:28.944 INFO:teuthology.orchestra.run.smithi089.stdout: "services_complete": [], 2024-09-21T04:11:28.944 INFO:teuthology.orchestra.run.smithi089.stdout: "progress": "1/23 daemons upgraded", 2024-09-21T04:11:28.944 INFO:teuthology.orchestra.run.smithi089.stdout: "message": "", 2024-09-21T04:11:28.944 INFO:teuthology.orchestra.run.smithi089.stdout: "is_paused": false 2024-09-21T04:11:28.944 INFO:teuthology.orchestra.run.smithi089.stdout:} 2024-09-21T04:11:29.314 INFO:teuthology.orchestra.run.smithi089.stdout: 5 68680 13.36 MB/sec execute 8 sec latency 30.406 ms 2024-09-21T04:11:29.739 INFO:teuthology.orchestra.run.smithi089.stdout:HEALTH_OK 2024-09-21T04:11:30.001 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:11:29 smithi089 ceph-mon[23733]: Adjusting osd_memory_target on smithi089 to 1935M 2024-09-21T04:11:30.001 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:11:29 smithi089 ceph-mon[23733]: Updating smithi089:/etc/ceph/ceph.conf 2024-09-21T04:11:30.001 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:11:29 smithi089 ceph-mon[23733]: Updating smithi195:/etc/ceph/ceph.conf 2024-09-21T04:11:30.001 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:11:29 smithi089 ceph-mon[23733]: pgmap v10: 97 pgs: 97 active+clean; 331 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 22 KiB/s rd, 6.1 MiB/s wr, 638 op/s 2024-09-21T04:11:30.001 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:11:29 smithi089 ceph-mon[23733]: Updating smithi195:/var/lib/ceph/1b9b79fc-77ce-11ef-baf5-efdc52797490/config/ceph.conf 2024-09-21T04:11:30.001 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:11:29 smithi089 ceph-mon[23733]: Updating smithi089:/var/lib/ceph/1b9b79fc-77ce-11ef-baf5-efdc52797490/config/ceph.conf 2024-09-21T04:11:30.001 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:11:29 smithi089 ceph-mon[23733]: Updating smithi195:/etc/ceph/ceph.client.admin.keyring 2024-09-21T04:11:30.001 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:11:29 smithi089 ceph-mon[23733]: Updating smithi089:/etc/ceph/ceph.client.admin.keyring 2024-09-21T04:11:30.002 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:11:29 smithi089 ceph-mon[23733]: Updating smithi195:/var/lib/ceph/1b9b79fc-77ce-11ef-baf5-efdc52797490/config/ceph.client.admin.keyring 2024-09-21T04:11:30.002 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:11:29 smithi089 ceph-mon[23733]: from='mgr.24407 ' entity='mgr.smithi195.cgotzm' 2024-09-21T04:11:30.002 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:11:29 smithi089 ceph-mon[23733]: Updating smithi089:/var/lib/ceph/1b9b79fc-77ce-11ef-baf5-efdc52797490/config/ceph.client.admin.keyring 2024-09-21T04:11:30.002 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:11:29 smithi089 ceph-mon[23733]: from='mgr.24407 ' entity='mgr.smithi195.cgotzm' 2024-09-21T04:11:30.002 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:11:29 smithi089 ceph-mon[23733]: from='client.14744 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T04:11:30.002 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:11:29 smithi089 ceph-mon[23733]: from='mgr.24407 ' entity='mgr.smithi195.cgotzm' 2024-09-21T04:11:30.002 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:11:29 smithi089 ceph-mon[23733]: from='mgr.24407 ' entity='mgr.smithi195.cgotzm' 2024-09-21T04:11:30.002 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:11:29 smithi089 ceph-mon[23733]: from='mgr.24407 ' entity='mgr.smithi195.cgotzm' 2024-09-21T04:11:30.002 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:11:29 smithi089 ceph-mon[23733]: from='mgr.24407 ' entity='mgr.smithi195.cgotzm' 2024-09-21T04:11:30.002 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:11:29 smithi089 ceph-mon[23733]: from='mgr.24407 ' entity='mgr.smithi195.cgotzm' 2024-09-21T04:11:30.002 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:11:29 smithi089 ceph-mon[23733]: from='mgr.24407 ' entity='mgr.smithi195.cgotzm' 2024-09-21T04:11:30.002 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:11:29 smithi089 ceph-mon[23733]: from='mgr.24407 ' entity='mgr.smithi195.cgotzm' 2024-09-21T04:11:30.002 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:11:29 smithi089 ceph-mon[23733]: from='mgr.24407 ' entity='mgr.smithi195.cgotzm' 2024-09-21T04:11:30.002 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:11:29 smithi089 ceph-mon[23733]: Reconfiguring alertmanager.smithi089 (dependencies changed)... 2024-09-21T04:11:30.002 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:11:29 smithi089 ceph-mon[23733]: Deploying daemon alertmanager.smithi089 on smithi089 2024-09-21T04:11:30.002 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:11:29 smithi089 ceph-mon[23733]: from='client.? 172.21.15.89:0/2577886764' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-21T04:11:30.109 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:11:29 smithi195 ceph-mon[25840]: Adjusting osd_memory_target on smithi089 to 1935M 2024-09-21T04:11:30.109 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:11:29 smithi195 ceph-mon[25840]: Updating smithi089:/etc/ceph/ceph.conf 2024-09-21T04:11:30.109 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:11:29 smithi195 ceph-mon[25840]: Updating smithi195:/etc/ceph/ceph.conf 2024-09-21T04:11:30.109 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:11:29 smithi195 ceph-mon[25840]: pgmap v10: 97 pgs: 97 active+clean; 331 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 22 KiB/s rd, 6.1 MiB/s wr, 638 op/s 2024-09-21T04:11:30.109 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:11:29 smithi195 ceph-mon[25840]: Updating smithi195:/var/lib/ceph/1b9b79fc-77ce-11ef-baf5-efdc52797490/config/ceph.conf 2024-09-21T04:11:30.109 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:11:29 smithi195 ceph-mon[25840]: Updating smithi089:/var/lib/ceph/1b9b79fc-77ce-11ef-baf5-efdc52797490/config/ceph.conf 2024-09-21T04:11:30.109 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:11:29 smithi195 ceph-mon[25840]: Updating smithi195:/etc/ceph/ceph.client.admin.keyring 2024-09-21T04:11:30.109 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:11:29 smithi195 ceph-mon[25840]: Updating smithi089:/etc/ceph/ceph.client.admin.keyring 2024-09-21T04:11:30.110 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:11:29 smithi195 ceph-mon[25840]: Updating smithi195:/var/lib/ceph/1b9b79fc-77ce-11ef-baf5-efdc52797490/config/ceph.client.admin.keyring 2024-09-21T04:11:30.110 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:11:29 smithi195 ceph-mon[25840]: from='mgr.24407 ' entity='mgr.smithi195.cgotzm' 2024-09-21T04:11:30.110 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:11:29 smithi195 ceph-mon[25840]: Updating smithi089:/var/lib/ceph/1b9b79fc-77ce-11ef-baf5-efdc52797490/config/ceph.client.admin.keyring 2024-09-21T04:11:30.110 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:11:29 smithi195 ceph-mon[25840]: from='mgr.24407 ' entity='mgr.smithi195.cgotzm' 2024-09-21T04:11:30.110 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:11:29 smithi195 ceph-mon[25840]: from='client.14744 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T04:11:30.110 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:11:29 smithi195 ceph-mon[25840]: from='mgr.24407 ' entity='mgr.smithi195.cgotzm' 2024-09-21T04:11:30.110 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:11:29 smithi195 ceph-mon[25840]: from='mgr.24407 ' entity='mgr.smithi195.cgotzm' 2024-09-21T04:11:30.110 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:11:29 smithi195 ceph-mon[25840]: from='mgr.24407 ' entity='mgr.smithi195.cgotzm' 2024-09-21T04:11:30.110 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:11:29 smithi195 ceph-mon[25840]: from='mgr.24407 ' entity='mgr.smithi195.cgotzm' 2024-09-21T04:11:30.110 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:11:29 smithi195 ceph-mon[25840]: from='mgr.24407 ' entity='mgr.smithi195.cgotzm' 2024-09-21T04:11:30.110 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:11:29 smithi195 ceph-mon[25840]: from='mgr.24407 ' entity='mgr.smithi195.cgotzm' 2024-09-21T04:11:30.110 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:11:29 smithi195 ceph-mon[25840]: from='mgr.24407 ' entity='mgr.smithi195.cgotzm' 2024-09-21T04:11:30.110 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:11:29 smithi195 ceph-mon[25840]: from='mgr.24407 ' entity='mgr.smithi195.cgotzm' 2024-09-21T04:11:30.110 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:11:29 smithi195 ceph-mon[25840]: Reconfiguring alertmanager.smithi089 (dependencies changed)... 2024-09-21T04:11:30.111 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:11:29 smithi195 ceph-mon[25840]: Deploying daemon alertmanager.smithi089 on smithi089 2024-09-21T04:11:30.111 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:11:29 smithi195 ceph-mon[25840]: from='client.? 172.21.15.89:0/2577886764' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-21T04:11:30.314 INFO:teuthology.orchestra.run.smithi089.stdout: 5 69364 13.14 MB/sec execute 9 sec latency 46.939 ms 2024-09-21T04:11:31.314 INFO:teuthology.orchestra.run.smithi089.stdout: 5 69945 13.10 MB/sec execute 10 sec latency 107.000 ms 2024-09-21T04:11:32.010 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:11:31 smithi089 ceph-mon[23733]: pgmap v11: 97 pgs: 97 active+clean; 332 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 26 KiB/s rd, 6.7 MiB/s wr, 742 op/s 2024-09-21T04:11:32.010 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:11:31 smithi089 ceph-mon[23733]: from='mgr.24407 172.21.15.195:0/751858916' entity='mgr.smithi195.cgotzm' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T04:11:32.109 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:11:31 smithi195 ceph-mon[25840]: pgmap v11: 97 pgs: 97 active+clean; 332 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 26 KiB/s rd, 6.7 MiB/s wr, 742 op/s 2024-09-21T04:11:32.109 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:11:31 smithi195 ceph-mon[25840]: from='mgr.24407 172.21.15.195:0/751858916' entity='mgr.smithi195.cgotzm' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T04:11:32.315 INFO:teuthology.orchestra.run.smithi089.stdout: 5 70702 14.85 MB/sec execute 11 sec latency 25.187 ms 2024-09-21T04:11:33.267 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:11:33 smithi089 ceph-mon[23733]: pgmap v12: 97 pgs: 97 active+clean; 314 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 24 KiB/s rd, 8.8 MiB/s wr, 926 op/s 2024-09-21T04:11:33.315 INFO:teuthology.orchestra.run.smithi089.stdout: 5 71239 15.34 MB/sec execute 12 sec latency 33.272 ms 2024-09-21T04:11:33.608 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:11:33 smithi195 ceph-mon[25840]: pgmap v12: 97 pgs: 97 active+clean; 314 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 24 KiB/s rd, 8.8 MiB/s wr, 926 op/s 2024-09-21T04:11:34.315 INFO:teuthology.orchestra.run.smithi089.stdout: 5 71686 14.47 MB/sec execute 13 sec latency 22.367 ms 2024-09-21T04:11:35.315 INFO:teuthology.orchestra.run.smithi089.stdout: 5 72136 14.16 MB/sec execute 14 sec latency 24.427 ms 2024-09-21T04:11:35.557 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:11:35 smithi089 ceph-mon[23733]: pgmap v13: 97 pgs: 97 active+clean; 314 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 9.4 KiB/s rd, 5.6 MiB/s wr, 598 op/s 2024-09-21T04:11:35.608 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:11:35 smithi195 ceph-mon[25840]: pgmap v13: 97 pgs: 97 active+clean; 314 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 9.4 KiB/s rd, 5.6 MiB/s wr, 598 op/s 2024-09-21T04:11:36.315 INFO:teuthology.orchestra.run.smithi089.stdout: 5 72740 14.20 MB/sec execute 15 sec latency 30.024 ms 2024-09-21T04:11:37.315 INFO:teuthology.orchestra.run.smithi089.stdout: 5 73557 14.24 MB/sec execute 16 sec latency 28.741 ms 2024-09-21T04:11:37.580 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:11:37 smithi089 ceph-mon[23733]: pgmap v14: 97 pgs: 97 active+clean; 325 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 14 KiB/s rd, 9.6 MiB/s wr, 941 op/s 2024-09-21T04:11:37.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:11:37 smithi195 ceph-mon[25840]: pgmap v14: 97 pgs: 97 active+clean; 325 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 14 KiB/s rd, 9.6 MiB/s wr, 941 op/s 2024-09-21T04:11:38.315 INFO:teuthology.orchestra.run.smithi089.stdout: 5 74322 15.29 MB/sec execute 17 sec latency 44.054 ms 2024-09-21T04:11:39.315 INFO:teuthology.orchestra.run.smithi089.stdout: 5 74761 15.50 MB/sec execute 18 sec latency 61.075 ms 2024-09-21T04:11:39.607 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:11:39 smithi089 ceph-mon[23733]: pgmap v15: 97 pgs: 97 active+clean; 325 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 9.1 KiB/s rd, 6.7 MiB/s wr, 636 op/s 2024-09-21T04:11:39.859 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:11:39 smithi195 ceph-mon[25840]: pgmap v15: 97 pgs: 97 active+clean; 325 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 9.1 KiB/s rd, 6.7 MiB/s wr, 636 op/s 2024-09-21T04:11:40.315 INFO:teuthology.orchestra.run.smithi089.stdout: 5 75065 14.87 MB/sec execute 19 sec latency 35.691 ms 2024-09-21T04:11:41.315 INFO:teuthology.orchestra.run.smithi089.stdout: 5 75473 14.36 MB/sec execute 20 sec latency 82.804 ms 2024-09-21T04:11:41.632 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:11:41 smithi089 ceph-mon[23733]: pgmap v16: 97 pgs: 97 active+clean; 332 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 13 KiB/s rd, 7.9 MiB/s wr, 818 op/s 2024-09-21T04:11:41.859 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:11:41 smithi195 ceph-mon[25840]: pgmap v16: 97 pgs: 97 active+clean; 332 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 13 KiB/s rd, 7.9 MiB/s wr, 818 op/s 2024-09-21T04:11:42.315 INFO:teuthology.orchestra.run.smithi089.stdout: 5 75876 14.22 MB/sec execute 21 sec latency 66.559 ms 2024-09-21T04:11:43.315 INFO:teuthology.orchestra.run.smithi089.stdout: 5 76460 14.08 MB/sec execute 22 sec latency 40.183 ms 2024-09-21T04:11:43.443 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:11:43 smithi089 ceph-mon[23733]: pgmap v17: 97 pgs: 97 active+clean; 316 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 7.8 KiB/s rd, 9.5 MiB/s wr, 854 op/s 2024-09-21T04:11:43.608 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:11:43 smithi195 ceph-mon[25840]: pgmap v17: 97 pgs: 97 active+clean; 316 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 7.8 KiB/s rd, 9.5 MiB/s wr, 854 op/s 2024-09-21T04:11:44.315 INFO:teuthology.orchestra.run.smithi089.stdout: 5 77024 13.96 MB/sec execute 23 sec latency 37.331 ms 2024-09-21T04:11:45.316 INFO:teuthology.orchestra.run.smithi089.stdout: 5 77448 14.16 MB/sec execute 24 sec latency 43.146 ms 2024-09-21T04:11:45.482 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:11:45 smithi089 ceph-mon[23733]: pgmap v18: 97 pgs: 97 active+clean; 316 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 7.8 KiB/s rd, 7.5 MiB/s wr, 664 op/s 2024-09-21T04:11:45.858 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:11:45 smithi195 ceph-mon[25840]: pgmap v18: 97 pgs: 97 active+clean; 316 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 7.8 KiB/s rd, 7.5 MiB/s wr, 664 op/s 2024-09-21T04:11:46.316 INFO:teuthology.orchestra.run.smithi089.stdout: 5 77968 14.41 MB/sec execute 25 sec latency 34.263 ms 2024-09-21T04:11:46.617 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:11:46 smithi089 ceph-mon[23733]: from='mgr.24407 172.21.15.195:0/751858916' entity='mgr.smithi195.cgotzm' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T04:11:46.858 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:11:46 smithi195 ceph-mon[25840]: from='mgr.24407 172.21.15.195:0/751858916' entity='mgr.smithi195.cgotzm' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T04:11:47.316 INFO:teuthology.orchestra.run.smithi089.stdout: 5 78358 14.44 MB/sec execute 26 sec latency 64.642 ms 2024-09-21T04:11:47.389 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:11:47 smithi089 ceph-mon[23733]: pgmap v19: 97 pgs: 97 active+clean; 314 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 13 KiB/s rd, 10 MiB/s wr, 953 op/s 2024-09-21T04:11:47.859 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:11:47 smithi195 ceph-mon[25840]: pgmap v19: 97 pgs: 97 active+clean; 314 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 13 KiB/s rd, 10 MiB/s wr, 953 op/s 2024-09-21T04:11:48.316 INFO:teuthology.orchestra.run.smithi089.stdout: 5 78723 14.03 MB/sec execute 27 sec latency 30.065 ms 2024-09-21T04:11:49.316 INFO:teuthology.orchestra.run.smithi089.stdout: 5 79013 13.65 MB/sec execute 28 sec latency 78.767 ms 2024-09-21T04:11:49.414 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:11:49 smithi089 ceph-mon[23733]: pgmap v20: 97 pgs: 97 active+clean; 314 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 8.4 KiB/s rd, 6.1 MiB/s wr, 610 op/s 2024-09-21T04:11:49.858 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:11:49 smithi195 ceph-mon[25840]: pgmap v20: 97 pgs: 97 active+clean; 314 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 8.4 KiB/s rd, 6.1 MiB/s wr, 610 op/s 2024-09-21T04:11:50.316 INFO:teuthology.orchestra.run.smithi089.stdout: 5 79281 13.44 MB/sec execute 29 sec latency 48.006 ms 2024-09-21T04:11:50.859 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:11:50 smithi195 ceph-mon[25840]: from='mgr.24407 ' entity='mgr.smithi195.cgotzm' 2024-09-21T04:11:50.859 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:11:50 smithi195 ceph-mon[25840]: from='mgr.24407 ' entity='mgr.smithi195.cgotzm' 2024-09-21T04:11:50.859 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:11:50 smithi195 ceph-mon[25840]: Reconfiguring grafana.smithi089 (dependencies changed)... 2024-09-21T04:11:50.859 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:11:50 smithi195 ceph-mon[25840]: Regenerating cephadm self-signed grafana TLS certificates 2024-09-21T04:11:50.859 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:11:50 smithi195 ceph-mon[25840]: from='mgr.24407 ' entity='mgr.smithi195.cgotzm' 2024-09-21T04:11:50.859 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:11:50 smithi195 ceph-mon[25840]: from='mgr.24407 ' entity='mgr.smithi195.cgotzm' 2024-09-21T04:11:50.859 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:11:50 smithi195 ceph-mon[25840]: from='mgr.24407 172.21.15.195:0/751858916' entity='mgr.smithi195.cgotzm' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-21T04:11:50.942 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:11:50 smithi089 ceph-mon[23733]: from='mgr.24407 ' entity='mgr.smithi195.cgotzm' 2024-09-21T04:11:50.943 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:11:50 smithi089 ceph-mon[23733]: from='mgr.24407 ' entity='mgr.smithi195.cgotzm' 2024-09-21T04:11:50.943 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:11:50 smithi089 ceph-mon[23733]: Reconfiguring grafana.smithi089 (dependencies changed)... 2024-09-21T04:11:50.943 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:11:50 smithi089 ceph-mon[23733]: Regenerating cephadm self-signed grafana TLS certificates 2024-09-21T04:11:50.943 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:11:50 smithi089 ceph-mon[23733]: from='mgr.24407 ' entity='mgr.smithi195.cgotzm' 2024-09-21T04:11:50.943 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:11:50 smithi089 ceph-mon[23733]: from='mgr.24407 ' entity='mgr.smithi195.cgotzm' 2024-09-21T04:11:50.943 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:11:50 smithi089 ceph-mon[23733]: from='mgr.24407 172.21.15.195:0/751858916' entity='mgr.smithi195.cgotzm' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-21T04:11:51.316 INFO:teuthology.orchestra.run.smithi089.stdout: 5 79656 13.36 MB/sec execute 30 sec latency 80.157 ms 2024-09-21T04:11:51.954 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:11:51 smithi089 ceph-mon[23733]: pgmap v21: 97 pgs: 97 active+clean; 319 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 7.2 MiB/s wr, 769 op/s 2024-09-21T04:11:51.954 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:11:51 smithi089 ceph-mon[23733]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-21T04:11:51.954 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:11:51 smithi089 ceph-mon[23733]: Reconfiguring daemon grafana.smithi089 on smithi089 2024-09-21T04:11:52.108 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:11:51 smithi195 ceph-mon[25840]: pgmap v21: 97 pgs: 97 active+clean; 319 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 7.2 MiB/s wr, 769 op/s 2024-09-21T04:11:52.109 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:11:51 smithi195 ceph-mon[25840]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-21T04:11:52.109 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:11:51 smithi195 ceph-mon[25840]: Reconfiguring daemon grafana.smithi089 on smithi089 2024-09-21T04:11:52.316 INFO:teuthology.orchestra.run.smithi089.stdout: 5 80255 13.17 MB/sec execute 31 sec latency 83.074 ms 2024-09-21T04:11:53.216 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:11:53 smithi089 ceph-mon[23733]: pgmap v22: 97 pgs: 97 active+clean; 317 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 8.2 KiB/s rd, 7.5 MiB/s wr, 662 op/s 2024-09-21T04:11:53.316 INFO:teuthology.orchestra.run.smithi089.stdout: 5 80605 13.08 MB/sec execute 32 sec latency 37.758 ms 2024-09-21T04:11:53.608 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:11:53 smithi195 ceph-mon[25840]: pgmap v22: 97 pgs: 97 active+clean; 317 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 8.2 KiB/s rd, 7.5 MiB/s wr, 662 op/s 2024-09-21T04:11:54.316 INFO:teuthology.orchestra.run.smithi089.stdout: 5 80905 13.04 MB/sec execute 33 sec latency 79.073 ms 2024-09-21T04:11:55.316 INFO:teuthology.orchestra.run.smithi089.stdout: 5 81292 13.17 MB/sec execute 34 sec latency 50.123 ms 2024-09-21T04:11:55.575 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:11:55 smithi089 ceph-mon[23733]: pgmap v23: 97 pgs: 97 active+clean; 317 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 8.2 KiB/s rd, 5.2 MiB/s wr, 522 op/s 2024-09-21T04:11:55.858 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:11:55 smithi195 ceph-mon[25840]: pgmap v23: 97 pgs: 97 active+clean; 317 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 8.2 KiB/s rd, 5.2 MiB/s wr, 522 op/s 2024-09-21T04:11:56.316 INFO:teuthology.orchestra.run.smithi089.stdout: 5 81705 13.26 MB/sec execute 35 sec latency 65.057 ms 2024-09-21T04:11:57.316 INFO:teuthology.orchestra.run.smithi089.stdout: 5 81953 13.12 MB/sec execute 36 sec latency 39.559 ms 2024-09-21T04:11:57.606 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:11:57 smithi089 ceph-mon[23733]: pgmap v24: 97 pgs: 97 active+clean; 320 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 7.5 MiB/s wr, 751 op/s 2024-09-21T04:11:57.606 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:11:57 smithi089 ceph-mon[23733]: from='mgr.24407 ' entity='mgr.smithi195.cgotzm' 2024-09-21T04:11:57.606 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:11:57 smithi089 ceph-mon[23733]: from='mgr.24407 ' entity='mgr.smithi195.cgotzm' 2024-09-21T04:11:57.859 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:11:57 smithi195 ceph-mon[25840]: pgmap v24: 97 pgs: 97 active+clean; 320 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 7.5 MiB/s wr, 751 op/s 2024-09-21T04:11:57.859 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:11:57 smithi195 ceph-mon[25840]: from='mgr.24407 ' entity='mgr.smithi195.cgotzm' 2024-09-21T04:11:57.859 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:11:57 smithi195 ceph-mon[25840]: from='mgr.24407 ' entity='mgr.smithi195.cgotzm' 2024-09-21T04:11:58.316 INFO:teuthology.orchestra.run.smithi089.stdout: 5 82205 12.81 MB/sec execute 37 sec latency 86.154 ms 2024-09-21T04:11:58.611 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:11:58 smithi089 ceph-mon[23733]: Reconfiguring node-exporter.smithi089 (dependencies changed)... 2024-09-21T04:11:58.611 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:11:58 smithi089 ceph-mon[23733]: Deploying daemon node-exporter.smithi089 on smithi089 2024-09-21T04:11:58.858 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:11:58 smithi195 ceph-mon[25840]: Reconfiguring node-exporter.smithi089 (dependencies changed)... 2024-09-21T04:11:58.859 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:11:58 smithi195 ceph-mon[25840]: Deploying daemon node-exporter.smithi089 on smithi089 2024-09-21T04:11:59.316 INFO:teuthology.orchestra.run.smithi089.stdout: 5 82564 12.58 MB/sec execute 38 sec latency 74.289 ms 2024-09-21T04:11:59.627 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:11:59 smithi089 ceph-mon[23733]: pgmap v25: 97 pgs: 97 active+clean; 320 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 6.9 KiB/s rd, 4.8 MiB/s wr, 463 op/s 2024-09-21T04:11:59.858 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:11:59 smithi195 ceph-mon[25840]: pgmap v25: 97 pgs: 97 active+clean; 320 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 6.9 KiB/s rd, 4.8 MiB/s wr, 463 op/s 2024-09-21T04:12:00.316 INFO:teuthology.orchestra.run.smithi089.stdout: 5 82856 12.48 MB/sec execute 39 sec latency 44.763 ms 2024-09-21T04:12:00.382 INFO:teuthology.orchestra.run.smithi089.stdout:true 2024-09-21T04:12:01.316 INFO:teuthology.orchestra.run.smithi089.stdout: 5 83399 12.50 MB/sec execute 40 sec latency 53.296 ms 2024-09-21T04:12:01.611 INFO:teuthology.orchestra.run.smithi089.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-21T04:12:01.611 INFO:teuthology.orchestra.run.smithi089.stdout:alertmanager.smithi089 smithi089 *:9093,9094 starting - - - - 2024-09-21T04:12:01.611 INFO:teuthology.orchestra.run.smithi089.stdout:crash.smithi089 smithi089 running (9m) 34s ago 9m 6409k - 17.2.0 e1d6a67b021e d1e0a8969f78 2024-09-21T04:12:01.612 INFO:teuthology.orchestra.run.smithi089.stdout:crash.smithi195 smithi195 running (8m) 39s ago 8m 6400k - 17.2.0 e1d6a67b021e 7e213ff76084 2024-09-21T04:12:01.612 INFO:teuthology.orchestra.run.smithi089.stdout:grafana.smithi089 smithi089 *:3000 starting - - - - 2024-09-21T04:12:01.612 INFO:teuthology.orchestra.run.smithi089.stdout:mds.foofs.smithi089.osufws smithi089 running (3m) 34s ago 3m 43.4M - 17.2.0 e1d6a67b021e 664d41859bea 2024-09-21T04:12:01.612 INFO:teuthology.orchestra.run.smithi089.stdout:mds.foofs.smithi195.hijxre smithi195 running (3m) 39s ago 3m 14.9M - 17.2.0 e1d6a67b021e 7bc0c6542d12 2024-09-21T04:12:01.612 INFO:teuthology.orchestra.run.smithi089.stdout:mgr.smithi089.srlyxf smithi089 *:9283 running (10m) 34s ago 10m 265M - 17.2.0 e1d6a67b021e 410bb00e916d 2024-09-21T04:12:01.612 INFO:teuthology.orchestra.run.smithi089.stdout:mgr.smithi195.cgotzm smithi195 *:8443,9283 running (69s) 39s ago 8m 499M - 19.3.0-5103-gf36283e7 48ed0a2cf89d 18c1e9ebdeda 2024-09-21T04:12:01.612 INFO:teuthology.orchestra.run.smithi089.stdout:mon.smithi089 smithi089 running (10m) 34s ago 10m 51.7M 2048M 17.2.0 e1d6a67b021e 1d2b3719e46e 2024-09-21T04:12:01.612 INFO:teuthology.orchestra.run.smithi089.stdout:mon.smithi195 smithi195 running (8m) 39s ago 8m 40.2M 2048M 17.2.0 e1d6a67b021e 7f86c5b5d21e 2024-09-21T04:12:01.612 INFO:teuthology.orchestra.run.smithi089.stdout:nfs.foo.0.0.smithi089.dcjmqc smithi089 *:2049 running (2m) 34s ago 2m 243M - 4.0 e1d6a67b021e 98550f94e773 2024-09-21T04:12:01.612 INFO:teuthology.orchestra.run.smithi089.stdout:nfs.foo.1.0.smithi195.vhoayp smithi195 *:2049 running (2m) 39s ago 2m 75.5M - 4.0 e1d6a67b021e c60a4040130a 2024-09-21T04:12:01.612 INFO:teuthology.orchestra.run.smithi089.stdout:node-exporter.smithi089 smithi089 *:9100 running (8m) 34s ago 8m 21.0M - 1.3.1 1dbe0e931976 5b166976fc1c 2024-09-21T04:12:01.612 INFO:teuthology.orchestra.run.smithi089.stdout:node-exporter.smithi195 smithi195 *:9100 running (8m) 39s ago 8m 18.4M - 1.3.1 1dbe0e931976 3448624c239b 2024-09-21T04:12:01.613 INFO:teuthology.orchestra.run.smithi089.stdout:osd.0 smithi089 running (7m) 34s ago 7m 283M 1935M 17.2.0 e1d6a67b021e db7c0e5020e7 2024-09-21T04:12:01.613 INFO:teuthology.orchestra.run.smithi089.stdout:osd.1 smithi089 running (7m) 34s ago 7m 206M 1935M 17.2.0 e1d6a67b021e f9ec697d08d0 2024-09-21T04:12:01.613 INFO:teuthology.orchestra.run.smithi089.stdout:osd.2 smithi089 running (6m) 34s ago 6m 191M 1935M 17.2.0 e1d6a67b021e cfa000948859 2024-09-21T04:12:01.613 INFO:teuthology.orchestra.run.smithi089.stdout:osd.3 smithi089 running (6m) 34s ago 6m 225M 1935M 17.2.0 e1d6a67b021e 31c610ed5815 2024-09-21T04:12:01.613 INFO:teuthology.orchestra.run.smithi089.stdout:osd.4 smithi195 running (5m) 39s ago 5m 246M 2704M 17.2.0 e1d6a67b021e f00e18a1f220 2024-09-21T04:12:01.613 INFO:teuthology.orchestra.run.smithi089.stdout:osd.5 smithi195 running (5m) 39s ago 5m 278M 2704M 17.2.0 e1d6a67b021e cecfd6ff1eee 2024-09-21T04:12:01.613 INFO:teuthology.orchestra.run.smithi089.stdout:osd.6 smithi195 running (5m) 39s ago 5m 246M 2704M 17.2.0 e1d6a67b021e 0d7db362cc23 2024-09-21T04:12:01.613 INFO:teuthology.orchestra.run.smithi089.stdout:osd.7 smithi195 running (4m) 39s ago 4m 328M 2704M 17.2.0 e1d6a67b021e 0109b3fea188 2024-09-21T04:12:01.613 INFO:teuthology.orchestra.run.smithi089.stdout:prometheus.smithi089 smithi089 *:9095 running (8m) 34s ago 8m 65.0M - 2.33.4 514e6a882f6e 9ed6ce8f2de3 2024-09-21T04:12:01.654 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:12:01 smithi089 ceph-mon[23733]: pgmap v26: 97 pgs: 97 active+clean; 321 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 9.8 KiB/s rd, 5.8 MiB/s wr, 569 op/s 2024-09-21T04:12:01.654 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:12:01 smithi089 ceph-mon[23733]: from='client.14752 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T04:12:01.654 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:12:01 smithi089 ceph-mon[23733]: from='mgr.24407 172.21.15.195:0/751858916' entity='mgr.smithi195.cgotzm' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T04:12:01.654 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:12:01 smithi089 ceph-mon[23733]: from='client.14756 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T04:12:01.859 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:12:01 smithi195 ceph-mon[25840]: pgmap v26: 97 pgs: 97 active+clean; 321 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 9.8 KiB/s rd, 5.8 MiB/s wr, 569 op/s 2024-09-21T04:12:01.859 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:12:01 smithi195 ceph-mon[25840]: from='client.14752 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T04:12:01.859 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:12:01 smithi195 ceph-mon[25840]: from='mgr.24407 172.21.15.195:0/751858916' entity='mgr.smithi195.cgotzm' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T04:12:01.859 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:12:01 smithi195 ceph-mon[25840]: from='client.14756 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T04:12:02.316 INFO:teuthology.orchestra.run.smithi089.stdout: 5 83950 12.34 MB/sec execute 41 sec latency 33.097 ms 2024-09-21T04:12:02.391 INFO:teuthology.orchestra.run.smithi089.stdout:{ 2024-09-21T04:12:02.391 INFO:teuthology.orchestra.run.smithi089.stdout: "mon": { 2024-09-21T04:12:02.391 INFO:teuthology.orchestra.run.smithi089.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-21T04:12:02.392 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2024-09-21T04:12:02.392 INFO:teuthology.orchestra.run.smithi089.stdout: "mgr": { 2024-09-21T04:12:02.392 INFO:teuthology.orchestra.run.smithi089.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 1, 2024-09-21T04:12:02.392 INFO:teuthology.orchestra.run.smithi089.stdout: "ceph version 19.3.0-5103-gf36283e7 (f36283e7c91cac1c01cb9a7c4ff21bd697dbd4f2) squid (dev)": 1 2024-09-21T04:12:02.392 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2024-09-21T04:12:02.392 INFO:teuthology.orchestra.run.smithi089.stdout: "osd": { 2024-09-21T04:12:02.392 INFO:teuthology.orchestra.run.smithi089.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-09-21T04:12:02.392 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2024-09-21T04:12:02.392 INFO:teuthology.orchestra.run.smithi089.stdout: "mds": { 2024-09-21T04:12:02.392 INFO:teuthology.orchestra.run.smithi089.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-21T04:12:02.392 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2024-09-21T04:12:02.392 INFO:teuthology.orchestra.run.smithi089.stdout: "overall": { 2024-09-21T04:12:02.392 INFO:teuthology.orchestra.run.smithi089.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 13, 2024-09-21T04:12:02.392 INFO:teuthology.orchestra.run.smithi089.stdout: "ceph version 19.3.0-5103-gf36283e7 (f36283e7c91cac1c01cb9a7c4ff21bd697dbd4f2) squid (dev)": 1 2024-09-21T04:12:02.393 INFO:teuthology.orchestra.run.smithi089.stdout: } 2024-09-21T04:12:02.393 INFO:teuthology.orchestra.run.smithi089.stdout:} 2024-09-21T04:12:02.662 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:12:02 smithi089 ceph-mon[23733]: from='client.14760 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T04:12:02.663 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:12:02 smithi089 ceph-mon[23733]: from='client.? 172.21.15.89:0/1910191382' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T04:12:02.858 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:12:02 smithi195 ceph-mon[25840]: from='client.14760 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T04:12:02.859 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:12:02 smithi195 ceph-mon[25840]: from='client.? 172.21.15.89:0/1910191382' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T04:12:03.002 INFO:teuthology.orchestra.run.smithi089.stdout:{ 2024-09-21T04:12:03.002 INFO:teuthology.orchestra.run.smithi089.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:2195e5f178be0a9390728e2a14088236d8fe6f82635f5c84418fe37a04457e00", 2024-09-21T04:12:03.002 INFO:teuthology.orchestra.run.smithi089.stdout: "in_progress": true, 2024-09-21T04:12:03.002 INFO:teuthology.orchestra.run.smithi089.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-21T04:12:03.002 INFO:teuthology.orchestra.run.smithi089.stdout: "services_complete": [], 2024-09-21T04:12:03.002 INFO:teuthology.orchestra.run.smithi089.stdout: "progress": "1/23 daemons upgraded", 2024-09-21T04:12:03.002 INFO:teuthology.orchestra.run.smithi089.stdout: "message": "", 2024-09-21T04:12:03.002 INFO:teuthology.orchestra.run.smithi089.stdout: "is_paused": false 2024-09-21T04:12:03.002 INFO:teuthology.orchestra.run.smithi089.stdout:} 2024-09-21T04:12:03.316 INFO:teuthology.orchestra.run.smithi089.stdout: 5 84229 12.31 MB/sec execute 42 sec latency 47.529 ms 2024-09-21T04:12:03.430 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:12:03 smithi089 ceph-mon[23733]: pgmap v27: 97 pgs: 97 active+clean; 320 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 6.4 KiB/s rd, 6.0 MiB/s wr, 507 op/s 2024-09-21T04:12:03.430 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:12:03 smithi089 ceph-mon[23733]: from='client.14768 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T04:12:03.561 INFO:teuthology.orchestra.run.smithi089.stdout:HEALTH_OK 2024-09-21T04:12:03.858 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:12:03 smithi195 ceph-mon[25840]: pgmap v27: 97 pgs: 97 active+clean; 320 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 6.4 KiB/s rd, 6.0 MiB/s wr, 507 op/s 2024-09-21T04:12:03.859 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:12:03 smithi195 ceph-mon[25840]: from='client.14768 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T04:12:04.316 INFO:teuthology.orchestra.run.smithi089.stdout: 5 84651 12.47 MB/sec execute 43 sec latency 39.947 ms 2024-09-21T04:12:04.477 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:12:04 smithi089 ceph-mon[23733]: from='client.? 172.21.15.89:0/4293471072' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-21T04:12:04.859 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:12:04 smithi195 ceph-mon[25840]: from='client.? 172.21.15.89:0/4293471072' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-21T04:12:05.316 INFO:teuthology.orchestra.run.smithi089.stdout: 5 85002 12.45 MB/sec execute 44 sec latency 73.621 ms 2024-09-21T04:12:06.014 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:12:05 smithi089 ceph-mon[23733]: pgmap v28: 97 pgs: 97 active+clean; 320 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 6.4 KiB/s rd, 4.5 MiB/s wr, 431 op/s 2024-09-21T04:12:06.014 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:12:05 smithi089 ceph-mon[23733]: from='mgr.24407 ' entity='mgr.smithi195.cgotzm' 2024-09-21T04:12:06.015 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:12:05 smithi089 ceph-mon[23733]: from='mgr.24407 ' entity='mgr.smithi195.cgotzm' 2024-09-21T04:12:06.015 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:12:05 smithi089 ceph-mon[23733]: Reconfiguring prometheus.smithi089 (dependencies changed)... 2024-09-21T04:12:06.015 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:12:05 smithi089 ceph-mon[23733]: Deploying daemon prometheus.smithi089 on smithi089 2024-09-21T04:12:06.109 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:12:05 smithi195 ceph-mon[25840]: pgmap v28: 97 pgs: 97 active+clean; 320 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 6.4 KiB/s rd, 4.5 MiB/s wr, 431 op/s 2024-09-21T04:12:06.109 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:12:05 smithi195 ceph-mon[25840]: from='mgr.24407 ' entity='mgr.smithi195.cgotzm' 2024-09-21T04:12:06.109 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:12:05 smithi195 ceph-mon[25840]: from='mgr.24407 ' entity='mgr.smithi195.cgotzm' 2024-09-21T04:12:06.109 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:12:05 smithi195 ceph-mon[25840]: Reconfiguring prometheus.smithi089 (dependencies changed)... 2024-09-21T04:12:06.109 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:12:05 smithi195 ceph-mon[25840]: Deploying daemon prometheus.smithi089 on smithi089 2024-09-21T04:12:06.316 INFO:teuthology.orchestra.run.smithi089.stdout: 5 85284 12.50 MB/sec execute 45 sec latency 78.226 ms 2024-09-21T04:12:07.316 INFO:teuthology.orchestra.run.smithi089.stdout: 5 85449 12.34 MB/sec execute 46 sec latency 73.144 ms 2024-09-21T04:12:07.823 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:12:07 smithi089 ceph-mon[23733]: pgmap v29: 97 pgs: 97 active+clean; 321 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 6.9 MiB/s wr, 670 op/s 2024-09-21T04:12:08.109 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:12:07 smithi195 ceph-mon[25840]: pgmap v29: 97 pgs: 97 active+clean; 321 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 6.9 MiB/s wr, 670 op/s 2024-09-21T04:12:08.316 INFO:teuthology.orchestra.run.smithi089.stdout: 5 85650 12.11 MB/sec execute 47 sec latency 49.303 ms 2024-09-21T04:12:09.316 INFO:teuthology.orchestra.run.smithi089.stdout: 5 85965 11.90 MB/sec execute 48 sec latency 50.551 ms 2024-09-21T04:12:09.889 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:12:09 smithi089 ceph-mon[23733]: pgmap v30: 97 pgs: 97 active+clean; 321 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 6.5 KiB/s rd, 4.6 MiB/s wr, 441 op/s 2024-09-21T04:12:10.109 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:12:09 smithi195 ceph-mon[25840]: pgmap v30: 97 pgs: 97 active+clean; 321 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 6.5 KiB/s rd, 4.6 MiB/s wr, 441 op/s 2024-09-21T04:12:10.316 INFO:teuthology.orchestra.run.smithi089.stdout: 5 86261 11.78 MB/sec execute 49 sec latency 42.372 ms 2024-09-21T04:12:11.316 INFO:teuthology.orchestra.run.smithi089.stdout: 5 86763 11.90 MB/sec execute 50 sec latency 34.585 ms 2024-09-21T04:12:11.929 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:12:11 smithi089 ceph-mon[23733]: pgmap v31: 97 pgs: 97 active+clean; 319 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 8.9 KiB/s rd, 5.1 MiB/s wr, 473 op/s 2024-09-21T04:12:12.108 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:12:11 smithi195 ceph-mon[25840]: pgmap v31: 97 pgs: 97 active+clean; 319 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 8.9 KiB/s rd, 5.1 MiB/s wr, 473 op/s 2024-09-21T04:12:12.317 INFO:teuthology.orchestra.run.smithi089.stdout: 5 87275 11.73 MB/sec execute 51 sec latency 52.261 ms 2024-09-21T04:12:13.189 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:12:13 smithi089 ceph-mon[23733]: pgmap v32: 97 pgs: 97 active+clean; 323 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 6.0 KiB/s rd, 5.5 MiB/s wr, 505 op/s 2024-09-21T04:12:13.317 INFO:teuthology.orchestra.run.smithi089.stdout: 5 87625 11.64 MB/sec execute 52 sec latency 32.378 ms 2024-09-21T04:12:13.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:12:13 smithi195 ceph-mon[25840]: pgmap v32: 97 pgs: 97 active+clean; 323 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 6.0 KiB/s rd, 5.5 MiB/s wr, 505 op/s 2024-09-21T04:12:14.317 INFO:teuthology.orchestra.run.smithi089.stdout: 5 88136 11.86 MB/sec execute 53 sec latency 80.681 ms 2024-09-21T04:12:15.317 INFO:teuthology.orchestra.run.smithi089.stdout: 5 88537 11.93 MB/sec execute 54 sec latency 32.487 ms 2024-09-21T04:12:15.523 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:12:15 smithi089 ceph-mon[23733]: pgmap v33: 97 pgs: 97 active+clean; 323 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 6.0 KiB/s rd, 4.2 MiB/s wr, 407 op/s 2024-09-21T04:12:15.858 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:12:15 smithi195 ceph-mon[25840]: pgmap v33: 97 pgs: 97 active+clean; 323 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 6.0 KiB/s rd, 4.2 MiB/s wr, 407 op/s 2024-09-21T04:12:16.317 INFO:teuthology.orchestra.run.smithi089.stdout: 5 88921 11.98 MB/sec execute 55 sec latency 46.813 ms 2024-09-21T04:12:16.546 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:12:16 smithi089 ceph-mon[23733]: from='mgr.24407 172.21.15.195:0/751858916' entity='mgr.smithi195.cgotzm' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T04:12:16.858 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:12:16 smithi195 ceph-mon[25840]: from='mgr.24407 172.21.15.195:0/751858916' entity='mgr.smithi195.cgotzm' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T04:12:17.317 INFO:teuthology.orchestra.run.smithi089.stdout: 5 89175 11.87 MB/sec execute 56 sec latency 40.803 ms 2024-09-21T04:12:17.442 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:12:17 smithi089 ceph-mon[23733]: pgmap v34: 97 pgs: 97 active+clean; 318 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 6.8 MiB/s wr, 666 op/s 2024-09-21T04:12:17.858 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:12:17 smithi195 ceph-mon[25840]: pgmap v34: 97 pgs: 97 active+clean; 318 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 6.8 MiB/s wr, 666 op/s 2024-09-21T04:12:18.317 INFO:teuthology.orchestra.run.smithi089.stdout: 5 89574 11.72 MB/sec execute 57 sec latency 37.779 ms 2024-09-21T04:12:19.317 INFO:teuthology.orchestra.run.smithi089.stdout: 5 89984 11.69 MB/sec execute 58 sec latency 43.088 ms 2024-09-21T04:12:19.473 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:12:19 smithi089 ceph-mon[23733]: pgmap v35: 97 pgs: 97 active+clean; 318 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 6.4 KiB/s rd, 4.4 MiB/s wr, 427 op/s 2024-09-21T04:12:19.858 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:12:19 smithi195 ceph-mon[25840]: pgmap v35: 97 pgs: 97 active+clean; 318 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 6.4 KiB/s rd, 4.4 MiB/s wr, 427 op/s 2024-09-21T04:12:20.317 INFO:teuthology.orchestra.run.smithi089.stdout: 5 90466 11.72 MB/sec execute 59 sec latency 46.862 ms 2024-09-21T04:12:21.317 INFO:teuthology.orchestra.run.smithi089.stdout: 5 90990 11.60 MB/sec execute 60 sec latency 36.023 ms 2024-09-21T04:12:21.512 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:12:21 smithi089 ceph-mon[23733]: pgmap v36: 97 pgs: 97 active+clean; 321 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 5.7 MiB/s wr, 623 op/s 2024-09-21T04:12:21.858 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:12:21 smithi195 ceph-mon[25840]: pgmap v36: 97 pgs: 97 active+clean; 321 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 5.7 MiB/s wr, 623 op/s 2024-09-21T04:12:22.317 INFO:teuthology.orchestra.run.smithi089.stdout: 5 91259 11.57 MB/sec execute 61 sec latency 43.877 ms 2024-09-21T04:12:23.289 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:12:23 smithi089 ceph-mon[23733]: pgmap v37: 97 pgs: 97 active+clean; 313 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 8.0 KiB/s rd, 6.1 MiB/s wr, 639 op/s 2024-09-21T04:12:23.317 INFO:teuthology.orchestra.run.smithi089.stdout: 5 91726 11.72 MB/sec execute 62 sec latency 43.073 ms 2024-09-21T04:12:23.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:12:23 smithi195 ceph-mon[25840]: pgmap v37: 97 pgs: 97 active+clean; 313 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 8.0 KiB/s rd, 6.1 MiB/s wr, 639 op/s 2024-09-21T04:12:24.317 INFO:teuthology.orchestra.run.smithi089.stdout: 5 92080 11.72 MB/sec execute 63 sec latency 66.619 ms 2024-09-21T04:12:25.317 INFO:teuthology.orchestra.run.smithi089.stdout: 5 92396 11.79 MB/sec execute 64 sec latency 47.654 ms 2024-09-21T04:12:25.580 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:12:25 smithi089 ceph-mon[23733]: pgmap v38: 97 pgs: 97 active+clean; 313 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 8.0 KiB/s rd, 4.8 MiB/s wr, 502 op/s 2024-09-21T04:12:25.859 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:12:25 smithi195 ceph-mon[25840]: pgmap v38: 97 pgs: 97 active+clean; 313 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 8.0 KiB/s rd, 4.8 MiB/s wr, 502 op/s 2024-09-21T04:12:26.317 INFO:teuthology.orchestra.run.smithi089.stdout: 5 92589 11.68 MB/sec execute 65 sec latency 62.422 ms 2024-09-21T04:12:27.317 INFO:teuthology.orchestra.run.smithi089.stdout: 5 92810 11.53 MB/sec execute 66 sec latency 86.687 ms 2024-09-21T04:12:27.598 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:12:27 smithi089 ceph-mon[23733]: pgmap v39: 97 pgs: 97 active+clean; 316 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 7.3 MiB/s wr, 719 op/s 2024-09-21T04:12:27.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:12:27 smithi195 ceph-mon[25840]: pgmap v39: 97 pgs: 97 active+clean; 316 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 7.3 MiB/s wr, 719 op/s 2024-09-21T04:12:28.317 INFO:teuthology.orchestra.run.smithi089.stdout: 5 93081 11.40 MB/sec execute 67 sec latency 65.083 ms 2024-09-21T04:12:29.317 INFO:teuthology.orchestra.run.smithi089.stdout: 5 93390 11.33 MB/sec execute 68 sec latency 64.294 ms 2024-09-21T04:12:29.425 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:12:29 smithi089 ceph-mon[23733]: from='mgr.24407 ' entity='mgr.smithi195.cgotzm' 2024-09-21T04:12:29.426 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:12:29 smithi089 ceph-mon[23733]: pgmap v40: 97 pgs: 97 active+clean; 316 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.7 MiB/s wr, 460 op/s 2024-09-21T04:12:29.426 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:12:29 smithi089 ceph-mon[23733]: from='mgr.24407 ' entity='mgr.smithi195.cgotzm' 2024-09-21T04:12:29.426 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:12:29 smithi089 ceph-mon[23733]: Reconfiguring node-exporter.smithi195 (dependencies changed)... 2024-09-21T04:12:29.426 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:12:29 smithi089 ceph-mon[23733]: Deploying daemon node-exporter.smithi195 on smithi195 2024-09-21T04:12:29.543 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:12:29 smithi195 ceph-mon[25840]: from='mgr.24407 ' entity='mgr.smithi195.cgotzm' 2024-09-21T04:12:29.543 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:12:29 smithi195 ceph-mon[25840]: pgmap v40: 97 pgs: 97 active+clean; 316 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.7 MiB/s wr, 460 op/s 2024-09-21T04:12:29.543 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:12:29 smithi195 ceph-mon[25840]: from='mgr.24407 ' entity='mgr.smithi195.cgotzm' 2024-09-21T04:12:29.543 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:12:29 smithi195 ceph-mon[25840]: Reconfiguring node-exporter.smithi195 (dependencies changed)... 2024-09-21T04:12:29.543 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:12:29 smithi195 ceph-mon[25840]: Deploying daemon node-exporter.smithi195 on smithi195 2024-09-21T04:12:30.317 INFO:teuthology.orchestra.run.smithi089.stdout: 5 93696 11.26 MB/sec execute 69 sec latency 60.784 ms 2024-09-21T04:12:31.317 INFO:teuthology.orchestra.run.smithi089.stdout: 5 94082 11.26 MB/sec execute 70 sec latency 68.170 ms 2024-09-21T04:12:31.418 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:12:31 smithi089 ceph-mon[23733]: pgmap v41: 97 pgs: 97 active+clean; 317 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 5.3 MiB/s wr, 548 op/s 2024-09-21T04:12:31.418 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:12:31 smithi089 ceph-mon[23733]: from='mgr.24407 172.21.15.195:0/751858916' entity='mgr.smithi195.cgotzm' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T04:12:31.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:12:31 smithi195 ceph-mon[25840]: pgmap v41: 97 pgs: 97 active+clean; 317 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 5.3 MiB/s wr, 548 op/s 2024-09-21T04:12:31.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:12:31 smithi195 ceph-mon[25840]: from='mgr.24407 172.21.15.195:0/751858916' entity='mgr.smithi195.cgotzm' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T04:12:32.317 INFO:teuthology.orchestra.run.smithi089.stdout: 5 94522 11.16 MB/sec execute 71 sec latency 59.642 ms 2024-09-21T04:12:33.264 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:12:33 smithi089 ceph-mon[23733]: pgmap v42: 97 pgs: 97 active+clean; 317 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 6.5 KiB/s rd, 5.1 MiB/s wr, 449 op/s 2024-09-21T04:12:33.317 INFO:teuthology.orchestra.run.smithi089.stdout: 5 94825 11.14 MB/sec execute 72 sec latency 58.909 ms 2024-09-21T04:12:33.496 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:12:33 smithi195 ceph-mon[25840]: pgmap v42: 97 pgs: 97 active+clean; 317 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 6.5 KiB/s rd, 5.1 MiB/s wr, 449 op/s 2024-09-21T04:12:34.173 INFO:teuthology.orchestra.run.smithi089.stdout:true 2024-09-21T04:12:34.317 INFO:teuthology.orchestra.run.smithi089.stdout: 5 95108 11.15 MB/sec execute 73 sec latency 74.412 ms 2024-09-21T04:12:35.064 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:12:34 smithi089 ceph-mon[23733]: from='mgr.24407 ' entity='mgr.smithi195.cgotzm' 2024-09-21T04:12:35.064 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:12:34 smithi089 ceph-mon[23733]: from='mgr.24407 ' entity='mgr.smithi195.cgotzm' 2024-09-21T04:12:35.064 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:12:34 smithi089 ceph-mon[23733]: from='mgr.24407 172.21.15.195:0/751858916' entity='mgr.smithi195.cgotzm' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-21T04:12:35.064 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:12:34 smithi089 ceph-mon[23733]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-21T04:12:35.064 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:12:34 smithi089 ceph-mon[23733]: from='mgr.24407 172.21.15.195:0/751858916' entity='mgr.smithi195.cgotzm' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi089.front.sepia.ceph.com:9093"}]: dispatch 2024-09-21T04:12:35.064 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:12:34 smithi089 ceph-mon[23733]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi089.front.sepia.ceph.com:9093"}]: dispatch 2024-09-21T04:12:35.064 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:12:34 smithi089 ceph-mon[23733]: from='mgr.24407 ' entity='mgr.smithi195.cgotzm' 2024-09-21T04:12:35.064 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:12:34 smithi089 ceph-mon[23733]: from='mgr.24407 172.21.15.195:0/751858916' entity='mgr.smithi195.cgotzm' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-21T04:12:35.064 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:12:34 smithi089 ceph-mon[23733]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-21T04:12:35.065 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:12:34 smithi089 ceph-mon[23733]: from='mgr.24407 172.21.15.195:0/751858916' entity='mgr.smithi195.cgotzm' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi089.front.sepia.ceph.com:3000"}]: dispatch 2024-09-21T04:12:35.065 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:12:34 smithi089 ceph-mon[23733]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi089.front.sepia.ceph.com:3000"}]: dispatch 2024-09-21T04:12:35.065 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:12:34 smithi089 ceph-mon[23733]: from='mgr.24407 ' entity='mgr.smithi195.cgotzm' 2024-09-21T04:12:35.065 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:12:34 smithi089 ceph-mon[23733]: from='mgr.24407 172.21.15.195:0/751858916' entity='mgr.smithi195.cgotzm' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-21T04:12:35.065 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:12:34 smithi089 ceph-mon[23733]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-21T04:12:35.065 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:12:34 smithi089 ceph-mon[23733]: from='mgr.24407 172.21.15.195:0/751858916' entity='mgr.smithi195.cgotzm' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi089.front.sepia.ceph.com:9095"}]: dispatch 2024-09-21T04:12:35.065 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:12:34 smithi089 ceph-mon[23733]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi089.front.sepia.ceph.com:9095"}]: dispatch 2024-09-21T04:12:35.065 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:12:34 smithi089 ceph-mon[23733]: from='mgr.24407 ' entity='mgr.smithi195.cgotzm' 2024-09-21T04:12:35.065 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:12:34 smithi089 ceph-mon[23733]: from='mgr.24407 172.21.15.195:0/751858916' entity='mgr.smithi195.cgotzm' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T04:12:35.065 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:12:34 smithi089 ceph-mon[23733]: from='mgr.24407 172.21.15.195:0/751858916' entity='mgr.smithi195.cgotzm' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-21T04:12:35.090 INFO:teuthology.orchestra.run.smithi089.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-21T04:12:35.090 INFO:teuthology.orchestra.run.smithi089.stdout:alertmanager.smithi089 smithi089 *:9093,9094 starting - - - - 2024-09-21T04:12:35.090 INFO:teuthology.orchestra.run.smithi089.stdout:crash.smithi089 smithi089 running (9m) 67s ago 9m 6409k - 17.2.0 e1d6a67b021e d1e0a8969f78 2024-09-21T04:12:35.090 INFO:teuthology.orchestra.run.smithi089.stdout:crash.smithi195 smithi195 running (9m) 72s ago 9m 6400k - 17.2.0 e1d6a67b021e 7e213ff76084 2024-09-21T04:12:35.090 INFO:teuthology.orchestra.run.smithi089.stdout:grafana.smithi089 smithi089 *:3000 starting - - - - 2024-09-21T04:12:35.090 INFO:teuthology.orchestra.run.smithi089.stdout:mds.foofs.smithi089.osufws smithi089 running (3m) 67s ago 3m 43.4M - 17.2.0 e1d6a67b021e 664d41859bea 2024-09-21T04:12:35.090 INFO:teuthology.orchestra.run.smithi089.stdout:mds.foofs.smithi195.hijxre smithi195 running (3m) 72s ago 3m 14.9M - 17.2.0 e1d6a67b021e 7bc0c6542d12 2024-09-21T04:12:35.090 INFO:teuthology.orchestra.run.smithi089.stdout:mgr.smithi089.srlyxf smithi089 *:9283 running (10m) 67s ago 10m 265M - 17.2.0 e1d6a67b021e 410bb00e916d 2024-09-21T04:12:35.090 INFO:teuthology.orchestra.run.smithi089.stdout:mgr.smithi195.cgotzm smithi195 *:8443,9283 running (103s) 72s ago 9m 499M - 19.3.0-5103-gf36283e7 48ed0a2cf89d 18c1e9ebdeda 2024-09-21T04:12:35.090 INFO:teuthology.orchestra.run.smithi089.stdout:mon.smithi089 smithi089 running (10m) 67s ago 11m 51.7M 2048M 17.2.0 e1d6a67b021e 1d2b3719e46e 2024-09-21T04:12:35.090 INFO:teuthology.orchestra.run.smithi089.stdout:mon.smithi195 smithi195 running (9m) 72s ago 9m 40.2M 2048M 17.2.0 e1d6a67b021e 7f86c5b5d21e 2024-09-21T04:12:35.090 INFO:teuthology.orchestra.run.smithi089.stdout:nfs.foo.0.0.smithi089.dcjmqc smithi089 *:2049 running (3m) 67s ago 3m 243M - 4.0 e1d6a67b021e 98550f94e773 2024-09-21T04:12:35.090 INFO:teuthology.orchestra.run.smithi089.stdout:nfs.foo.1.0.smithi195.vhoayp smithi195 *:2049 running (3m) 72s ago 3m 75.5M - 4.0 e1d6a67b021e c60a4040130a 2024-09-21T04:12:35.091 INFO:teuthology.orchestra.run.smithi089.stdout:node-exporter.smithi089 smithi089 *:9100 starting - - - - 2024-09-21T04:12:35.091 INFO:teuthology.orchestra.run.smithi089.stdout:node-exporter.smithi195 smithi195 *:9100 starting - - - - 2024-09-21T04:12:35.091 INFO:teuthology.orchestra.run.smithi089.stdout:osd.0 smithi089 running (8m) 67s ago 8m 283M 1935M 17.2.0 e1d6a67b021e db7c0e5020e7 2024-09-21T04:12:35.091 INFO:teuthology.orchestra.run.smithi089.stdout:osd.1 smithi089 running (7m) 67s ago 7m 206M 1935M 17.2.0 e1d6a67b021e f9ec697d08d0 2024-09-21T04:12:35.091 INFO:teuthology.orchestra.run.smithi089.stdout:osd.2 smithi089 running (7m) 67s ago 7m 191M 1935M 17.2.0 e1d6a67b021e cfa000948859 2024-09-21T04:12:35.091 INFO:teuthology.orchestra.run.smithi089.stdout:osd.3 smithi089 running (6m) 67s ago 6m 225M 1935M 17.2.0 e1d6a67b021e 31c610ed5815 2024-09-21T04:12:35.091 INFO:teuthology.orchestra.run.smithi089.stdout:osd.4 smithi195 running (6m) 72s ago 6m 246M 2704M 17.2.0 e1d6a67b021e f00e18a1f220 2024-09-21T04:12:35.091 INFO:teuthology.orchestra.run.smithi089.stdout:osd.5 smithi195 running (6m) 72s ago 6m 278M 2704M 17.2.0 e1d6a67b021e cecfd6ff1eee 2024-09-21T04:12:35.091 INFO:teuthology.orchestra.run.smithi089.stdout:osd.6 smithi195 running (5m) 72s ago 5m 246M 2704M 17.2.0 e1d6a67b021e 0d7db362cc23 2024-09-21T04:12:35.091 INFO:teuthology.orchestra.run.smithi089.stdout:osd.7 smithi195 running (5m) 72s ago 5m 328M 2704M 17.2.0 e1d6a67b021e 0109b3fea188 2024-09-21T04:12:35.091 INFO:teuthology.orchestra.run.smithi089.stdout:prometheus.smithi089 smithi089 *:9095 starting - - - - 2024-09-21T04:12:35.108 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:12:34 smithi195 ceph-mon[25840]: from='mgr.24407 ' entity='mgr.smithi195.cgotzm' 2024-09-21T04:12:35.109 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:12:34 smithi195 ceph-mon[25840]: from='mgr.24407 ' entity='mgr.smithi195.cgotzm' 2024-09-21T04:12:35.109 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:12:34 smithi195 ceph-mon[25840]: from='mgr.24407 172.21.15.195:0/751858916' entity='mgr.smithi195.cgotzm' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-21T04:12:35.109 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:12:34 smithi195 ceph-mon[25840]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-21T04:12:35.109 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:12:34 smithi195 ceph-mon[25840]: from='mgr.24407 172.21.15.195:0/751858916' entity='mgr.smithi195.cgotzm' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi089.front.sepia.ceph.com:9093"}]: dispatch 2024-09-21T04:12:35.109 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:12:34 smithi195 ceph-mon[25840]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi089.front.sepia.ceph.com:9093"}]: dispatch 2024-09-21T04:12:35.109 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:12:34 smithi195 ceph-mon[25840]: from='mgr.24407 ' entity='mgr.smithi195.cgotzm' 2024-09-21T04:12:35.109 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:12:34 smithi195 ceph-mon[25840]: from='mgr.24407 172.21.15.195:0/751858916' entity='mgr.smithi195.cgotzm' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-21T04:12:35.109 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:12:34 smithi195 ceph-mon[25840]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-21T04:12:35.109 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:12:34 smithi195 ceph-mon[25840]: from='mgr.24407 172.21.15.195:0/751858916' entity='mgr.smithi195.cgotzm' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi089.front.sepia.ceph.com:3000"}]: dispatch 2024-09-21T04:12:35.109 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:12:34 smithi195 ceph-mon[25840]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi089.front.sepia.ceph.com:3000"}]: dispatch 2024-09-21T04:12:35.109 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:12:34 smithi195 ceph-mon[25840]: from='mgr.24407 ' entity='mgr.smithi195.cgotzm' 2024-09-21T04:12:35.109 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:12:34 smithi195 ceph-mon[25840]: from='mgr.24407 172.21.15.195:0/751858916' entity='mgr.smithi195.cgotzm' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-21T04:12:35.109 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:12:34 smithi195 ceph-mon[25840]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-21T04:12:35.110 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:12:34 smithi195 ceph-mon[25840]: from='mgr.24407 172.21.15.195:0/751858916' entity='mgr.smithi195.cgotzm' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi089.front.sepia.ceph.com:9095"}]: dispatch 2024-09-21T04:12:35.110 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:12:34 smithi195 ceph-mon[25840]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi089.front.sepia.ceph.com:9095"}]: dispatch 2024-09-21T04:12:35.110 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:12:34 smithi195 ceph-mon[25840]: from='mgr.24407 ' entity='mgr.smithi195.cgotzm' 2024-09-21T04:12:35.110 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:12:34 smithi195 ceph-mon[25840]: from='mgr.24407 172.21.15.195:0/751858916' entity='mgr.smithi195.cgotzm' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T04:12:35.110 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:12:34 smithi195 ceph-mon[25840]: from='mgr.24407 172.21.15.195:0/751858916' entity='mgr.smithi195.cgotzm' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-21T04:12:35.317 INFO:teuthology.orchestra.run.smithi089.stdout: 5 95462 11.22 MB/sec execute 74 sec latency 58.423 ms 2024-09-21T04:12:35.670 INFO:teuthology.orchestra.run.smithi089.stdout:{ 2024-09-21T04:12:35.670 INFO:teuthology.orchestra.run.smithi089.stdout: "mon": { 2024-09-21T04:12:35.670 INFO:teuthology.orchestra.run.smithi089.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-21T04:12:35.670 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2024-09-21T04:12:35.670 INFO:teuthology.orchestra.run.smithi089.stdout: "mgr": { 2024-09-21T04:12:35.670 INFO:teuthology.orchestra.run.smithi089.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 1, 2024-09-21T04:12:35.670 INFO:teuthology.orchestra.run.smithi089.stdout: "ceph version 19.3.0-5103-gf36283e7 (f36283e7c91cac1c01cb9a7c4ff21bd697dbd4f2) squid (dev)": 1 2024-09-21T04:12:35.670 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2024-09-21T04:12:35.670 INFO:teuthology.orchestra.run.smithi089.stdout: "osd": { 2024-09-21T04:12:35.670 INFO:teuthology.orchestra.run.smithi089.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-09-21T04:12:35.671 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2024-09-21T04:12:35.671 INFO:teuthology.orchestra.run.smithi089.stdout: "mds": { 2024-09-21T04:12:35.671 INFO:teuthology.orchestra.run.smithi089.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-21T04:12:35.671 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2024-09-21T04:12:35.671 INFO:teuthology.orchestra.run.smithi089.stdout: "overall": { 2024-09-21T04:12:35.671 INFO:teuthology.orchestra.run.smithi089.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 13, 2024-09-21T04:12:35.671 INFO:teuthology.orchestra.run.smithi089.stdout: "ceph version 19.3.0-5103-gf36283e7 (f36283e7c91cac1c01cb9a7c4ff21bd697dbd4f2) squid (dev)": 1 2024-09-21T04:12:35.671 INFO:teuthology.orchestra.run.smithi089.stdout: } 2024-09-21T04:12:35.671 INFO:teuthology.orchestra.run.smithi089.stdout:} 2024-09-21T04:12:36.085 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:12:35 smithi089 ceph-mon[23733]: from='client.14776 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T04:12:36.085 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:12:35 smithi089 ceph-mon[23733]: Upgrade: Need to upgrade myself (mgr.smithi195.cgotzm) 2024-09-21T04:12:36.085 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:12:35 smithi089 ceph-mon[23733]: Upgrade: Need to upgrade myself (mgr.smithi195.cgotzm) 2024-09-21T04:12:36.085 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:12:35 smithi089 ceph-mon[23733]: pgmap v43: 97 pgs: 97 active+clean; 317 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 6.5 KiB/s rd, 4.3 MiB/s wr, 401 op/s 2024-09-21T04:12:36.085 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:12:35 smithi089 ceph-mon[23733]: from='mon.? -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-21T04:12:36.085 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:12:35 smithi089 ceph-mon[23733]: from='client.24495 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T04:12:36.085 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:12:35 smithi089 ceph-mon[23733]: from='mgr.24407 172.21.15.195:0/751858916' entity='mgr.smithi195.cgotzm' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-21T04:12:36.085 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:12:35 smithi089 ceph-mon[23733]: from='mon.? -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-21T04:12:36.085 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:12:35 smithi089 ceph-mon[23733]: from='client.14784 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T04:12:36.085 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:12:35 smithi089 ceph-mon[23733]: from='mgr.24407 172.21.15.195:0/751858916' entity='mgr.smithi195.cgotzm' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-21T04:12:36.085 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:12:35 smithi089 ceph-mon[23733]: from='client.? 172.21.15.89:0/3322250561' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T04:12:36.305 INFO:teuthology.orchestra.run.smithi089.stdout:{ 2024-09-21T04:12:36.306 INFO:teuthology.orchestra.run.smithi089.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:2195e5f178be0a9390728e2a14088236d8fe6f82635f5c84418fe37a04457e00", 2024-09-21T04:12:36.306 INFO:teuthology.orchestra.run.smithi089.stdout: "in_progress": true, 2024-09-21T04:12:36.306 INFO:teuthology.orchestra.run.smithi089.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-21T04:12:36.306 INFO:teuthology.orchestra.run.smithi089.stdout: "services_complete": [], 2024-09-21T04:12:36.306 INFO:teuthology.orchestra.run.smithi089.stdout: "progress": "1/23 daemons upgraded", 2024-09-21T04:12:36.306 INFO:teuthology.orchestra.run.smithi089.stdout: "message": "Currently upgrading mgr daemons", 2024-09-21T04:12:36.306 INFO:teuthology.orchestra.run.smithi089.stdout: "is_paused": false 2024-09-21T04:12:36.306 INFO:teuthology.orchestra.run.smithi089.stdout:} 2024-09-21T04:12:36.317 INFO:teuthology.orchestra.run.smithi089.stdout: 5 95933 11.34 MB/sec execute 75 sec latency 43.541 ms 2024-09-21T04:12:36.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:12:35 smithi195 ceph-mon[25840]: from='client.14776 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T04:12:36.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:12:35 smithi195 ceph-mon[25840]: Upgrade: Need to upgrade myself (mgr.smithi195.cgotzm) 2024-09-21T04:12:36.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:12:35 smithi195 ceph-mon[25840]: Upgrade: Need to upgrade myself (mgr.smithi195.cgotzm) 2024-09-21T04:12:36.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:12:35 smithi195 ceph-mon[25840]: pgmap v43: 97 pgs: 97 active+clean; 317 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 6.5 KiB/s rd, 4.3 MiB/s wr, 401 op/s 2024-09-21T04:12:36.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:12:35 smithi195 ceph-mon[25840]: from='mon.? -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-21T04:12:36.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:12:35 smithi195 ceph-mon[25840]: from='client.24495 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T04:12:36.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:12:35 smithi195 ceph-mon[25840]: from='mgr.24407 172.21.15.195:0/751858916' entity='mgr.smithi195.cgotzm' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-21T04:12:36.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:12:35 smithi195 ceph-mon[25840]: from='mon.? -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-21T04:12:36.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:12:35 smithi195 ceph-mon[25840]: from='client.14784 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T04:12:36.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:12:35 smithi195 ceph-mon[25840]: from='mgr.24407 172.21.15.195:0/751858916' entity='mgr.smithi195.cgotzm' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-21T04:12:36.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:12:35 smithi195 ceph-mon[25840]: from='client.? 172.21.15.89:0/3322250561' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T04:12:36.916 INFO:teuthology.orchestra.run.smithi089.stdout:HEALTH_OK 2024-09-21T04:12:37.103 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:12:36 smithi089 ceph-mon[23733]: from='mon.? -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-21T04:12:37.103 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:12:36 smithi089 ceph-mon[23733]: from='mgr.24407 ' entity='mgr.smithi195.cgotzm' 2024-09-21T04:12:37.103 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:12:36 smithi089 ceph-mon[23733]: from='mgr.24407 172.21.15.195:0/751858916' entity='mgr.smithi195.cgotzm' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi089.srlyxf", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-21T04:12:37.103 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:12:36 smithi089 ceph-mon[23733]: from='mgr.24407 ' entity='mgr.smithi195.cgotzm' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi089.srlyxf", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-21T04:12:37.103 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:12:36 smithi089 ceph-mon[23733]: from='mgr.24407 172.21.15.195:0/751858916' entity='mgr.smithi195.cgotzm' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-21T04:12:37.103 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:12:36 smithi089 ceph-mon[23733]: from='mgr.24407 172.21.15.195:0/751858916' entity='mgr.smithi195.cgotzm' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T04:12:37.187 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:12:36 smithi195 ceph-mon[25840]: from='mon.? -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-21T04:12:37.187 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:12:36 smithi195 ceph-mon[25840]: from='mgr.24407 ' entity='mgr.smithi195.cgotzm' 2024-09-21T04:12:37.187 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:12:36 smithi195 ceph-mon[25840]: from='mgr.24407 172.21.15.195:0/751858916' entity='mgr.smithi195.cgotzm' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi089.srlyxf", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-21T04:12:37.187 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:12:36 smithi195 ceph-mon[25840]: from='mgr.24407 ' entity='mgr.smithi195.cgotzm' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi089.srlyxf", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-21T04:12:37.187 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:12:36 smithi195 ceph-mon[25840]: from='mgr.24407 172.21.15.195:0/751858916' entity='mgr.smithi195.cgotzm' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-21T04:12:37.188 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:12:36 smithi195 ceph-mon[25840]: from='mgr.24407 172.21.15.195:0/751858916' entity='mgr.smithi195.cgotzm' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T04:12:37.317 INFO:teuthology.orchestra.run.smithi089.stdout: 5 96190 11.25 MB/sec execute 76 sec latency 80.364 ms 2024-09-21T04:12:38.126 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:12:37 smithi089 ceph-mon[23733]: from='client.14792 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T04:12:38.126 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:12:37 smithi089 ceph-mon[23733]: pgmap v44: 97 pgs: 97 active+clean; 314 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 9.1 KiB/s rd, 6.4 MiB/s wr, 601 op/s 2024-09-21T04:12:38.126 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:12:37 smithi089 ceph-mon[23733]: Upgrade: Updating mgr.smithi089.srlyxf 2024-09-21T04:12:38.126 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:12:37 smithi089 ceph-mon[23733]: Deploying daemon mgr.smithi089.srlyxf on smithi089 2024-09-21T04:12:38.126 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:12:37 smithi089 ceph-mon[23733]: from='client.? 172.21.15.89:0/4067633938' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-21T04:12:38.317 INFO:teuthology.orchestra.run.smithi089.stdout: 5 96440 11.13 MB/sec execute 77 sec latency 43.268 ms 2024-09-21T04:12:38.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:12:37 smithi195 ceph-mon[25840]: from='client.14792 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T04:12:38.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:12:37 smithi195 ceph-mon[25840]: pgmap v44: 97 pgs: 97 active+clean; 314 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 9.1 KiB/s rd, 6.4 MiB/s wr, 601 op/s 2024-09-21T04:12:38.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:12:37 smithi195 ceph-mon[25840]: Upgrade: Updating mgr.smithi089.srlyxf 2024-09-21T04:12:38.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:12:37 smithi195 ceph-mon[25840]: Deploying daemon mgr.smithi089.srlyxf on smithi089 2024-09-21T04:12:38.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:12:37 smithi195 ceph-mon[25840]: from='client.? 172.21.15.89:0/4067633938' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-21T04:12:39.318 INFO:teuthology.orchestra.run.smithi089.stdout: 5 96746 11.04 MB/sec execute 78 sec latency 49.979 ms 2024-09-21T04:12:39.921 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:12:39 smithi089 ceph-mon[23733]: pgmap v45: 97 pgs: 97 active+clean; 314 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 6.3 KiB/s rd, 3.8 MiB/s wr, 384 op/s 2024-09-21T04:12:40.317 INFO:teuthology.orchestra.run.smithi089.stdout: 5 97084 11.04 MB/sec execute 79 sec latency 59.374 ms 2024-09-21T04:12:40.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:12:39 smithi195 ceph-mon[25840]: pgmap v45: 97 pgs: 97 active+clean; 314 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 6.3 KiB/s rd, 3.8 MiB/s wr, 384 op/s 2024-09-21T04:12:41.317 INFO:teuthology.orchestra.run.smithi089.stdout: 5 97503 11.03 MB/sec execute 80 sec latency 39.773 ms 2024-09-21T04:12:41.971 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:12:41 smithi089 ceph-mon[23733]: pgmap v46: 97 pgs: 97 active+clean; 318 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 9.4 KiB/s rd, 4.7 MiB/s wr, 476 op/s 2024-09-21T04:12:42.318 INFO:teuthology.orchestra.run.smithi089.stdout: 5 97914 10.94 MB/sec execute 81 sec latency 51.634 ms 2024-09-21T04:12:42.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:12:41 smithi195 ceph-mon[25840]: pgmap v46: 97 pgs: 97 active+clean; 318 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 9.4 KiB/s rd, 4.7 MiB/s wr, 476 op/s 2024-09-21T04:12:43.318 INFO:teuthology.orchestra.run.smithi089.stdout: 5 98251 10.91 MB/sec execute 82 sec latency 38.339 ms 2024-09-21T04:12:43.746 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:12:43 smithi195 ceph-mon[25840]: pgmap v47: 97 pgs: 97 active+clean; 320 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 5.7 KiB/s rd, 5.4 MiB/s wr, 506 op/s 2024-09-21T04:12:43.746 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:12:43 smithi195 ceph-mon[25840]: from='mgr.24407 ' entity='mgr.smithi195.cgotzm' 2024-09-21T04:12:43.746 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:12:43 smithi195 ceph-mon[25840]: from='mgr.24407 ' entity='mgr.smithi195.cgotzm' 2024-09-21T04:12:43.746 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:12:43 smithi195 ceph-mon[25840]: from='mgr.24407 172.21.15.195:0/751858916' entity='mgr.smithi195.cgotzm' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T04:12:43.746 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:12:43 smithi195 ceph-mon[25840]: from='mgr.24407 ' entity='mgr.smithi195.cgotzm' 2024-09-21T04:12:43.746 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:12:43 smithi195 ceph-mon[25840]: from='mgr.24407 ' entity='mgr.smithi195.cgotzm' 2024-09-21T04:12:43.763 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:12:43 smithi089 ceph-mon[23733]: pgmap v47: 97 pgs: 97 active+clean; 320 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 5.7 KiB/s rd, 5.4 MiB/s wr, 506 op/s 2024-09-21T04:12:43.763 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:12:43 smithi089 ceph-mon[23733]: from='mgr.24407 ' entity='mgr.smithi195.cgotzm' 2024-09-21T04:12:43.763 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:12:43 smithi089 ceph-mon[23733]: from='mgr.24407 ' entity='mgr.smithi195.cgotzm' 2024-09-21T04:12:43.763 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:12:43 smithi089 ceph-mon[23733]: from='mgr.24407 172.21.15.195:0/751858916' entity='mgr.smithi195.cgotzm' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T04:12:43.763 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:12:43 smithi089 ceph-mon[23733]: from='mgr.24407 ' entity='mgr.smithi195.cgotzm' 2024-09-21T04:12:43.764 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:12:43 smithi089 ceph-mon[23733]: from='mgr.24407 ' entity='mgr.smithi195.cgotzm' 2024-09-21T04:12:44.318 INFO:teuthology.orchestra.run.smithi089.stdout: 5 98474 10.87 MB/sec execute 83 sec latency 94.120 ms 2024-09-21T04:12:45.318 INFO:teuthology.orchestra.run.smithi089.stdout: 5 98915 11.00 MB/sec execute 84 sec latency 45.569 ms 2024-09-21T04:12:45.816 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:12:45 smithi089 ceph-mon[23733]: pgmap v48: 97 pgs: 97 active+clean; 320 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 5.7 KiB/s rd, 4.3 MiB/s wr, 409 op/s 2024-09-21T04:12:46.001 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:12:45 smithi195 ceph-mon[25840]: pgmap v48: 97 pgs: 97 active+clean; 320 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 5.7 KiB/s rd, 4.3 MiB/s wr, 409 op/s 2024-09-21T04:12:46.318 INFO:teuthology.orchestra.run.smithi089.stdout: 5 99244 10.99 MB/sec execute 85 sec latency 46.083 ms 2024-09-21T04:12:46.871 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:12:46 smithi089 ceph-mon[23733]: from='mgr.24407 172.21.15.195:0/751858916' entity='mgr.smithi195.cgotzm' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T04:12:46.871 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:12:46 smithi089 ceph-mon[23733]: from='mgr.24407 ' entity='mgr.smithi195.cgotzm' 2024-09-21T04:12:46.871 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:12:46 smithi089 ceph-mon[23733]: from='mgr.24407 ' entity='mgr.smithi195.cgotzm' 2024-09-21T04:12:47.109 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:12:46 smithi195 ceph-mon[25840]: from='mgr.24407 172.21.15.195:0/751858916' entity='mgr.smithi195.cgotzm' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T04:12:47.109 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:12:46 smithi195 ceph-mon[25840]: from='mgr.24407 ' entity='mgr.smithi195.cgotzm' 2024-09-21T04:12:47.109 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:12:46 smithi195 ceph-mon[25840]: from='mgr.24407 ' entity='mgr.smithi195.cgotzm' 2024-09-21T04:12:47.318 INFO:teuthology.orchestra.run.smithi089.stdout: 5 99524 11.01 MB/sec execute 86 sec latency 50.515 ms 2024-09-21T04:12:47.918 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:12:47 smithi089 ceph-mon[23733]: pgmap v49: 97 pgs: 97 active+clean; 318 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 8.7 KiB/s rd, 6.4 MiB/s wr, 619 op/s 2024-09-21T04:12:48.077 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:12:47 smithi195 ceph-mon[25840]: pgmap v49: 97 pgs: 97 active+clean; 318 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 8.7 KiB/s rd, 6.4 MiB/s wr, 619 op/s 2024-09-21T04:12:48.318 INFO:teuthology.orchestra.run.smithi089.stdout: 5 99698 10.93 MB/sec execute 87 sec latency 44.378 ms 2024-09-21T04:12:48.947 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:12:48 smithi089 ceph-mon[23733]: mgrmap e28: smithi195.cgotzm(active, since 94s), standbys: smithi089.srlyxf 2024-09-21T04:12:49.108 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:12:48 smithi195 ceph-mon[25840]: mgrmap e28: smithi195.cgotzm(active, since 94s), standbys: smithi089.srlyxf 2024-09-21T04:12:49.318 INFO:teuthology.orchestra.run.smithi089.stdout: 5 99899 10.82 MB/sec execute 88 sec latency 59.295 ms 2024-09-21T04:12:49.987 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:12:49 smithi089 ceph-mon[23733]: pgmap v50: 97 pgs: 97 active+clean; 318 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.3 MiB/s wr, 419 op/s 2024-09-21T04:12:49.987 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:12:49 smithi089 ceph-mon[23733]: Standby manager daemon smithi089.srlyxf restarted 2024-09-21T04:12:49.987 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:12:49 smithi089 ceph-mon[23733]: Standby manager daemon smithi089.srlyxf started 2024-09-21T04:12:49.987 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:12:49 smithi089 ceph-mon[23733]: from='mgr.? 172.21.15.89:0/1173442431' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi089.srlyxf/crt"}]: dispatch 2024-09-21T04:12:49.987 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:12:49 smithi089 ceph-mon[23733]: from='mgr.? 172.21.15.89:0/1173442431' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-21T04:12:49.987 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:12:49 smithi089 ceph-mon[23733]: from='mgr.? 172.21.15.89:0/1173442431' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi089.srlyxf/key"}]: dispatch 2024-09-21T04:12:49.987 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:12:49 smithi089 ceph-mon[23733]: from='mgr.? 172.21.15.89:0/1173442431' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-21T04:12:50.318 INFO:teuthology.orchestra.run.smithi089.stdout: 5 100121 10.72 MB/sec execute 89 sec latency 65.352 ms 2024-09-21T04:12:50.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:12:49 smithi195 ceph-mon[25840]: pgmap v50: 97 pgs: 97 active+clean; 318 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.3 MiB/s wr, 419 op/s 2024-09-21T04:12:50.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:12:49 smithi195 ceph-mon[25840]: Standby manager daemon smithi089.srlyxf restarted 2024-09-21T04:12:50.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:12:49 smithi195 ceph-mon[25840]: Standby manager daemon smithi089.srlyxf started 2024-09-21T04:12:50.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:12:49 smithi195 ceph-mon[25840]: from='mgr.? 172.21.15.89:0/1173442431' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi089.srlyxf/crt"}]: dispatch 2024-09-21T04:12:50.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:12:49 smithi195 ceph-mon[25840]: from='mgr.? 172.21.15.89:0/1173442431' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-21T04:12:50.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:12:49 smithi195 ceph-mon[25840]: from='mgr.? 172.21.15.89:0/1173442431' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi089.srlyxf/key"}]: dispatch 2024-09-21T04:12:50.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:12:49 smithi195 ceph-mon[25840]: from='mgr.? 172.21.15.89:0/1173442431' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-21T04:12:51.078 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:12:50 smithi089 ceph-mon[23733]: mgrmap e29: smithi195.cgotzm(active, since 96s), standbys: smithi089.srlyxf 2024-09-21T04:12:51.318 INFO:teuthology.orchestra.run.smithi089.stdout: 5 100336 10.64 MB/sec execute 90 sec latency 63.284 ms 2024-09-21T04:12:51.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:12:50 smithi195 ceph-mon[25840]: mgrmap e29: smithi195.cgotzm(active, since 96s), standbys: smithi089.srlyxf 2024-09-21T04:12:52.084 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:12:51 smithi089 ceph-mon[23733]: pgmap v51: 97 pgs: 97 active+clean; 314 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 8.2 KiB/s rd, 4.6 MiB/s wr, 434 op/s 2024-09-21T04:12:52.085 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:12:51 smithi089 ceph-mon[23733]: from='mgr.24407 ' entity='mgr.smithi195.cgotzm' 2024-09-21T04:12:52.085 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:12:51 smithi089 ceph-mon[23733]: from='mgr.24407 ' entity='mgr.smithi195.cgotzm' 2024-09-21T04:12:52.318 INFO:teuthology.orchestra.run.smithi089.stdout: 5 100748 10.64 MB/sec execute 91 sec latency 54.004 ms 2024-09-21T04:12:52.358 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:12:51 smithi195 ceph-mon[25840]: pgmap v51: 97 pgs: 97 active+clean; 314 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 8.2 KiB/s rd, 4.6 MiB/s wr, 434 op/s 2024-09-21T04:12:52.358 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:12:51 smithi195 ceph-mon[25840]: from='mgr.24407 ' entity='mgr.smithi195.cgotzm' 2024-09-21T04:12:52.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:12:51 smithi195 ceph-mon[25840]: from='mgr.24407 ' entity='mgr.smithi195.cgotzm' 2024-09-21T04:12:53.101 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:12:52 smithi089 ceph-mon[23733]: from='mgr.24407 ' entity='mgr.smithi195.cgotzm' 2024-09-21T04:12:53.101 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:12:52 smithi089 ceph-mon[23733]: from='mgr.24407 ' entity='mgr.smithi195.cgotzm' 2024-09-21T04:12:53.318 INFO:teuthology.orchestra.run.smithi089.stdout: 5 101241 10.65 MB/sec execute 92 sec latency 44.457 ms 2024-09-21T04:12:53.358 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:12:52 smithi195 ceph-mon[25840]: from='mgr.24407 ' entity='mgr.smithi195.cgotzm' 2024-09-21T04:12:53.358 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:12:52 smithi195 ceph-mon[25840]: from='mgr.24407 ' entity='mgr.smithi195.cgotzm' 2024-09-21T04:12:54.128 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:12:53 smithi089 ceph-mon[23733]: pgmap v52: 97 pgs: 97 active+clean; 323 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 5.2 KiB/s rd, 5.0 MiB/s wr, 473 op/s 2024-09-21T04:12:54.318 INFO:teuthology.orchestra.run.smithi089.stdout: 5 101608 10.57 MB/sec execute 93 sec latency 45.186 ms 2024-09-21T04:12:54.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:12:53 smithi195 ceph-mon[25840]: pgmap v52: 97 pgs: 97 active+clean; 323 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 5.2 KiB/s rd, 5.0 MiB/s wr, 473 op/s 2024-09-21T04:12:55.318 INFO:teuthology.orchestra.run.smithi089.stdout: 5 101974 10.60 MB/sec execute 94 sec latency 44.158 ms 2024-09-21T04:12:56.181 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:12:55 smithi089 ceph-mon[23733]: pgmap v53: 97 pgs: 97 active+clean; 323 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 5.2 KiB/s rd, 3.7 MiB/s wr, 356 op/s 2024-09-21T04:12:56.318 INFO:teuthology.orchestra.run.smithi089.stdout: 5 102408 10.68 MB/sec execute 95 sec latency 38.290 ms 2024-09-21T04:12:56.358 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:12:55 smithi195 ceph-mon[25840]: pgmap v53: 97 pgs: 97 active+clean; 323 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 5.2 KiB/s rd, 3.7 MiB/s wr, 356 op/s 2024-09-21T04:12:57.318 INFO:teuthology.orchestra.run.smithi089.stdout: 5 102865 10.75 MB/sec execute 96 sec latency 53.198 ms 2024-09-21T04:12:58.260 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:12:58 smithi089 ceph-mon[23733]: pgmap v54: 97 pgs: 97 active+clean; 314 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 9.2 KiB/s rd, 5.5 MiB/s wr, 583 op/s 2024-09-21T04:12:58.260 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:12:58 smithi089 ceph-mon[23733]: from='mgr.24407 ' entity='mgr.smithi195.cgotzm' 2024-09-21T04:12:58.260 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:12:58 smithi089 ceph-mon[23733]: from='mgr.24407 ' entity='mgr.smithi195.cgotzm' 2024-09-21T04:12:58.261 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:12:58 smithi089 ceph-mon[23733]: from='mgr.24407 172.21.15.195:0/751858916' entity='mgr.smithi195.cgotzm' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T04:12:58.261 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:12:58 smithi089 ceph-mon[23733]: from='mgr.24407 172.21.15.195:0/751858916' entity='mgr.smithi195.cgotzm' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T04:12:58.261 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:12:58 smithi089 ceph-mon[23733]: from='mgr.24407 ' entity='mgr.smithi195.cgotzm' 2024-09-21T04:12:58.261 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:12:58 smithi089 ceph-mon[23733]: from='mgr.24407 ' entity='mgr.smithi195.cgotzm' 2024-09-21T04:12:58.318 INFO:teuthology.orchestra.run.smithi089.stdout: 5 103157 10.77 MB/sec execute 97 sec latency 69.712 ms 2024-09-21T04:12:58.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:12:58 smithi195 ceph-mon[25840]: pgmap v54: 97 pgs: 97 active+clean; 314 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 9.2 KiB/s rd, 5.5 MiB/s wr, 583 op/s 2024-09-21T04:12:58.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:12:58 smithi195 ceph-mon[25840]: from='mgr.24407 ' entity='mgr.smithi195.cgotzm' 2024-09-21T04:12:58.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:12:58 smithi195 ceph-mon[25840]: from='mgr.24407 ' entity='mgr.smithi195.cgotzm' 2024-09-21T04:12:58.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:12:58 smithi195 ceph-mon[25840]: from='mgr.24407 172.21.15.195:0/751858916' entity='mgr.smithi195.cgotzm' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T04:12:58.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:12:58 smithi195 ceph-mon[25840]: from='mgr.24407 172.21.15.195:0/751858916' entity='mgr.smithi195.cgotzm' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T04:12:58.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:12:58 smithi195 ceph-mon[25840]: from='mgr.24407 ' entity='mgr.smithi195.cgotzm' 2024-09-21T04:12:58.360 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:12:58 smithi195 ceph-mon[25840]: from='mgr.24407 ' entity='mgr.smithi195.cgotzm' 2024-09-21T04:12:59.017 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:12:59 smithi089 ceph-mon[23733]: from='mgr.24407 172.21.15.195:0/751858916' entity='mgr.smithi195.cgotzm' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T04:12:59.017 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:12:59 smithi089 ceph-mon[23733]: from='mgr.24407 172.21.15.195:0/751858916' entity='mgr.smithi195.cgotzm' cmd=[{"prefix": "mgr fail", "who": "smithi195.cgotzm"}]: dispatch 2024-09-21T04:12:59.017 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:12:59 smithi089 ceph-mon[23733]: from='mgr.24407 ' entity='mgr.smithi195.cgotzm' cmd=[{"prefix": "mgr fail", "who": "smithi195.cgotzm"}]: dispatch 2024-09-21T04:12:59.017 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:12:59 smithi089 ceph-mon[23733]: osdmap e72: 8 total, 8 up, 8 in 2024-09-21T04:12:59.278 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:12:59 smithi089 ceph-mon[23733]: Standby manager daemon smithi195.cgotzm started 2024-09-21T04:12:59.318 INFO:teuthology.orchestra.run.smithi089.stdout: 5 103439 10.69 MB/sec execute 98 sec latency 41.024 ms 2024-09-21T04:12:59.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:12:59 smithi195 ceph-mon[25840]: from='mgr.24407 172.21.15.195:0/751858916' entity='mgr.smithi195.cgotzm' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T04:12:59.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:12:59 smithi195 ceph-mon[25840]: from='mgr.24407 172.21.15.195:0/751858916' entity='mgr.smithi195.cgotzm' cmd=[{"prefix": "mgr fail", "who": "smithi195.cgotzm"}]: dispatch 2024-09-21T04:12:59.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:12:59 smithi195 ceph-mon[25840]: from='mgr.24407 ' entity='mgr.smithi195.cgotzm' cmd=[{"prefix": "mgr fail", "who": "smithi195.cgotzm"}]: dispatch 2024-09-21T04:12:59.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:12:59 smithi195 ceph-mon[25840]: osdmap e72: 8 total, 8 up, 8 in 2024-09-21T04:12:59.360 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:12:59 smithi195 ceph-mon[25840]: Standby manager daemon smithi195.cgotzm started 2024-09-21T04:13:00.078 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:00 smithi089 ceph-mon[23733]: from='mgr.24407 ' entity='mgr.smithi195.cgotzm' cmd='[{"prefix": "mgr fail", "who": "smithi195.cgotzm"}]': finished 2024-09-21T04:13:00.078 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:00 smithi089 ceph-mon[23733]: mgrmap e30: smithi089.srlyxf(active, starting, since 0.695839s), standbys: smithi195.cgotzm 2024-09-21T04:13:00.078 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:00 smithi089 ceph-mon[23733]: from='mgr.14800 172.21.15.89:0/1173442431' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "mon metadata", "id": "smithi089"}]: dispatch 2024-09-21T04:13:00.078 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:00 smithi089 ceph-mon[23733]: from='mgr.14800 172.21.15.89:0/1173442431' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "mon metadata", "id": "smithi195"}]: dispatch 2024-09-21T04:13:00.078 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:00 smithi089 ceph-mon[23733]: from='mgr.14800 172.21.15.89:0/1173442431' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi089.osufws"}]: dispatch 2024-09-21T04:13:00.078 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:00 smithi089 ceph-mon[23733]: from='mgr.14800 172.21.15.89:0/1173442431' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi195.hijxre"}]: dispatch 2024-09-21T04:13:00.318 INFO:teuthology.orchestra.run.smithi089.stdout: 5 103742 10.61 MB/sec execute 99 sec latency 48.736 ms 2024-09-21T04:13:00.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:13:00 smithi195 ceph-mon[25840]: from='mgr.24407 ' entity='mgr.smithi195.cgotzm' cmd='[{"prefix": "mgr fail", "who": "smithi195.cgotzm"}]': finished 2024-09-21T04:13:00.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:13:00 smithi195 ceph-mon[25840]: mgrmap e30: smithi089.srlyxf(active, starting, since 0.695839s), standbys: smithi195.cgotzm 2024-09-21T04:13:00.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:13:00 smithi195 ceph-mon[25840]: from='mgr.14800 172.21.15.89:0/1173442431' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "mon metadata", "id": "smithi089"}]: dispatch 2024-09-21T04:13:00.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:13:00 smithi195 ceph-mon[25840]: from='mgr.14800 172.21.15.89:0/1173442431' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "mon metadata", "id": "smithi195"}]: dispatch 2024-09-21T04:13:00.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:13:00 smithi195 ceph-mon[25840]: from='mgr.14800 172.21.15.89:0/1173442431' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi089.osufws"}]: dispatch 2024-09-21T04:13:00.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:13:00 smithi195 ceph-mon[25840]: from='mgr.14800 172.21.15.89:0/1173442431' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi195.hijxre"}]: dispatch 2024-09-21T04:13:01.318 INFO:teuthology.orchestra.run.smithi089.stdout: 5 103995 10.54 MB/sec execute 100 sec latency 48.712 ms 2024-09-21T04:13:02.318 INFO:teuthology.orchestra.run.smithi089.stdout: 5 104316 10.53 MB/sec execute 101 sec latency 92.526 ms 2024-09-21T04:13:03.318 INFO:teuthology.orchestra.run.smithi089.stdout: 5 104744 10.53 MB/sec execute 102 sec latency 46.380 ms 2024-09-21T04:13:04.318 INFO:teuthology.orchestra.run.smithi089.stdout: 5 105134 10.47 MB/sec execute 103 sec latency 54.987 ms 2024-09-21T04:13:04.907 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:04 smithi089 ceph-mon[23733]: from='mgr.? 172.21.15.195:0/887597780' entity='mgr.smithi195.cgotzm' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi195.cgotzm/crt"}]: dispatch 2024-09-21T04:13:04.907 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:04 smithi089 ceph-mon[23733]: from='mgr.? 172.21.15.195:0/887597780' entity='mgr.smithi195.cgotzm' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-21T04:13:04.907 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:04 smithi089 ceph-mon[23733]: from='mgr.? 172.21.15.195:0/887597780' entity='mgr.smithi195.cgotzm' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi195.cgotzm/key"}]: dispatch 2024-09-21T04:13:04.907 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:04 smithi089 ceph-mon[23733]: from='mgr.? 172.21.15.195:0/887597780' entity='mgr.smithi195.cgotzm' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-21T04:13:04.907 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:04 smithi089 ceph-mon[23733]: Standby manager daemon smithi195.cgotzm restarted 2024-09-21T04:13:04.907 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:04 smithi089 ceph-mon[23733]: Standby manager daemon smithi195.cgotzm started 2024-09-21T04:13:05.109 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:13:04 smithi195 ceph-mon[25840]: from='mgr.? 172.21.15.195:0/887597780' entity='mgr.smithi195.cgotzm' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi195.cgotzm/crt"}]: dispatch 2024-09-21T04:13:05.109 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:13:04 smithi195 ceph-mon[25840]: from='mgr.? 172.21.15.195:0/887597780' entity='mgr.smithi195.cgotzm' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-21T04:13:05.109 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:13:04 smithi195 ceph-mon[25840]: from='mgr.? 172.21.15.195:0/887597780' entity='mgr.smithi195.cgotzm' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi195.cgotzm/key"}]: dispatch 2024-09-21T04:13:05.109 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:13:04 smithi195 ceph-mon[25840]: from='mgr.? 172.21.15.195:0/887597780' entity='mgr.smithi195.cgotzm' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-21T04:13:05.109 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:13:04 smithi195 ceph-mon[25840]: Standby manager daemon smithi195.cgotzm restarted 2024-09-21T04:13:05.109 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:13:04 smithi195 ceph-mon[25840]: Standby manager daemon smithi195.cgotzm started 2024-09-21T04:13:05.318 INFO:teuthology.orchestra.run.smithi089.stdout: 5 105504 10.49 MB/sec execute 104 sec latency 34.824 ms 2024-09-21T04:13:05.917 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:05 smithi089 ceph-mon[23733]: mgrmap e31: smithi089.srlyxf(active, starting, since 6s), standbys: smithi195.cgotzm 2024-09-21T04:13:06.278 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:13:05 smithi195 ceph-mon[25840]: mgrmap e31: smithi089.srlyxf(active, starting, since 6s), standbys: smithi195.cgotzm 2024-09-21T04:13:06.318 INFO:teuthology.orchestra.run.smithi089.stdout: 5 106085 10.64 MB/sec execute 105 sec latency 38.854 ms 2024-09-21T04:13:07.207 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:06 smithi089 ceph-mon[23733]: Active manager daemon smithi089.srlyxf restarted 2024-09-21T04:13:07.207 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:06 smithi089 ceph-mon[23733]: Activating manager daemon smithi089.srlyxf 2024-09-21T04:13:07.207 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:06 smithi089 ceph-mon[23733]: osdmap e73: 8 total, 8 up, 8 in 2024-09-21T04:13:07.314 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:13:06 smithi195 ceph-mon[25840]: Active manager daemon smithi089.srlyxf restarted 2024-09-21T04:13:07.314 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:13:06 smithi195 ceph-mon[25840]: Activating manager daemon smithi089.srlyxf 2024-09-21T04:13:07.314 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:13:06 smithi195 ceph-mon[25840]: osdmap e73: 8 total, 8 up, 8 in 2024-09-21T04:13:07.318 INFO:teuthology.orchestra.run.smithi089.stdout: 5 106588 10.73 MB/sec execute 106 sec latency 38.613 ms 2024-09-21T04:13:07.966 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:07 smithi089 ceph-mon[23733]: mgrmap e32: smithi089.srlyxf(active, starting, since 0.058925s), standbys: smithi195.cgotzm 2024-09-21T04:13:07.966 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:07 smithi089 ceph-mon[23733]: from='mgr.14806 172.21.15.89:0/3152262325' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "mon metadata", "id": "smithi089"}]: dispatch 2024-09-21T04:13:07.966 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:07 smithi089 ceph-mon[23733]: from='mgr.14806 172.21.15.89:0/3152262325' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "mon metadata", "id": "smithi195"}]: dispatch 2024-09-21T04:13:07.967 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:07 smithi089 ceph-mon[23733]: from='mgr.14806 172.21.15.89:0/3152262325' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi089.osufws"}]: dispatch 2024-09-21T04:13:07.967 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:07 smithi089 ceph-mon[23733]: from='mgr.14806 172.21.15.89:0/3152262325' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi195.hijxre"}]: dispatch 2024-09-21T04:13:07.967 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:07 smithi089 ceph-mon[23733]: from='mgr.14806 172.21.15.89:0/3152262325' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "mgr metadata", "who": "smithi089.srlyxf", "id": "smithi089.srlyxf"}]: dispatch 2024-09-21T04:13:07.967 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:07 smithi089 ceph-mon[23733]: from='mgr.14806 172.21.15.89:0/3152262325' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "mgr metadata", "who": "smithi195.cgotzm", "id": "smithi195.cgotzm"}]: dispatch 2024-09-21T04:13:07.967 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:07 smithi089 ceph-mon[23733]: from='mgr.14806 172.21.15.89:0/3152262325' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-21T04:13:07.967 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:07 smithi089 ceph-mon[23733]: from='mgr.14806 172.21.15.89:0/3152262325' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-21T04:13:07.967 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:07 smithi089 ceph-mon[23733]: from='mgr.14806 172.21.15.89:0/3152262325' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-21T04:13:07.967 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:07 smithi089 ceph-mon[23733]: from='mgr.14806 172.21.15.89:0/3152262325' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-21T04:13:07.967 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:07 smithi089 ceph-mon[23733]: from='mgr.14806 172.21.15.89:0/3152262325' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-21T04:13:07.967 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:07 smithi089 ceph-mon[23733]: from='mgr.14806 172.21.15.89:0/3152262325' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-21T04:13:07.967 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:07 smithi089 ceph-mon[23733]: from='mgr.14806 172.21.15.89:0/3152262325' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-21T04:13:07.967 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:07 smithi089 ceph-mon[23733]: from='mgr.14806 172.21.15.89:0/3152262325' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-21T04:13:07.967 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:07 smithi089 ceph-mon[23733]: from='mgr.14806 172.21.15.89:0/3152262325' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-21T04:13:07.967 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:07 smithi089 ceph-mon[23733]: from='mgr.14806 172.21.15.89:0/3152262325' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-21T04:13:07.967 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:07 smithi089 ceph-mon[23733]: from='mgr.14806 172.21.15.89:0/3152262325' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-21T04:13:07.968 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:07 smithi089 ceph-mon[23733]: Manager daemon smithi089.srlyxf is now available 2024-09-21T04:13:07.968 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:07 smithi089 ceph-mon[23733]: from='mgr.14806 172.21.15.89:0/3152262325' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T04:13:07.968 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:07 smithi089 ceph-mon[23733]: from='mgr.14806 172.21.15.89:0/3152262325' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-21T04:13:07.968 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:07 smithi089 ceph-mon[23733]: from='mgr.14806 172.21.15.89:0/3152262325' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T04:13:07.968 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:07 smithi089 ceph-mon[23733]: from='mgr.14806 172.21.15.89:0/3152262325' entity='mgr.smithi089.srlyxf' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi089.srlyxf/mirror_snapshot_schedule"}]: dispatch 2024-09-21T04:13:07.968 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:07 smithi089 ceph-mon[23733]: from='mgr.14806 172.21.15.89:0/3152262325' entity='mgr.smithi089.srlyxf' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi089.srlyxf/trash_purge_schedule"}]: dispatch 2024-09-21T04:13:07.968 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:07 smithi089 ceph-mon[23733]: mgrmap e33: smithi089.srlyxf(active, since 1.06847s), standbys: smithi195.cgotzm 2024-09-21T04:13:08.077 INFO:teuthology.orchestra.run.smithi089.stdout:true 2024-09-21T04:13:08.318 INFO:teuthology.orchestra.run.smithi089.stdout: 5 106840 10.67 MB/sec execute 107 sec latency 63.959 ms 2024-09-21T04:13:08.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:13:07 smithi195 ceph-mon[25840]: mgrmap e32: smithi089.srlyxf(active, starting, since 0.058925s), standbys: smithi195.cgotzm 2024-09-21T04:13:08.360 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:13:07 smithi195 ceph-mon[25840]: from='mgr.14806 172.21.15.89:0/3152262325' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "mon metadata", "id": "smithi089"}]: dispatch 2024-09-21T04:13:08.360 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:13:07 smithi195 ceph-mon[25840]: from='mgr.14806 172.21.15.89:0/3152262325' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "mon metadata", "id": "smithi195"}]: dispatch 2024-09-21T04:13:08.360 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:13:07 smithi195 ceph-mon[25840]: from='mgr.14806 172.21.15.89:0/3152262325' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi089.osufws"}]: dispatch 2024-09-21T04:13:08.360 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:13:07 smithi195 ceph-mon[25840]: from='mgr.14806 172.21.15.89:0/3152262325' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi195.hijxre"}]: dispatch 2024-09-21T04:13:08.360 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:13:07 smithi195 ceph-mon[25840]: from='mgr.14806 172.21.15.89:0/3152262325' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "mgr metadata", "who": "smithi089.srlyxf", "id": "smithi089.srlyxf"}]: dispatch 2024-09-21T04:13:08.360 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:13:07 smithi195 ceph-mon[25840]: from='mgr.14806 172.21.15.89:0/3152262325' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "mgr metadata", "who": "smithi195.cgotzm", "id": "smithi195.cgotzm"}]: dispatch 2024-09-21T04:13:08.360 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:13:07 smithi195 ceph-mon[25840]: from='mgr.14806 172.21.15.89:0/3152262325' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-21T04:13:08.360 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:13:07 smithi195 ceph-mon[25840]: from='mgr.14806 172.21.15.89:0/3152262325' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-21T04:13:08.360 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:13:07 smithi195 ceph-mon[25840]: from='mgr.14806 172.21.15.89:0/3152262325' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-21T04:13:08.361 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:13:07 smithi195 ceph-mon[25840]: from='mgr.14806 172.21.15.89:0/3152262325' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-21T04:13:08.361 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:13:07 smithi195 ceph-mon[25840]: from='mgr.14806 172.21.15.89:0/3152262325' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-21T04:13:08.361 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:13:07 smithi195 ceph-mon[25840]: from='mgr.14806 172.21.15.89:0/3152262325' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-21T04:13:08.361 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:13:07 smithi195 ceph-mon[25840]: from='mgr.14806 172.21.15.89:0/3152262325' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-21T04:13:08.361 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:13:07 smithi195 ceph-mon[25840]: from='mgr.14806 172.21.15.89:0/3152262325' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-21T04:13:08.361 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:13:07 smithi195 ceph-mon[25840]: from='mgr.14806 172.21.15.89:0/3152262325' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-21T04:13:08.361 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:13:07 smithi195 ceph-mon[25840]: from='mgr.14806 172.21.15.89:0/3152262325' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-21T04:13:08.361 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:13:07 smithi195 ceph-mon[25840]: from='mgr.14806 172.21.15.89:0/3152262325' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-21T04:13:08.361 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:13:07 smithi195 ceph-mon[25840]: Manager daemon smithi089.srlyxf is now available 2024-09-21T04:13:08.361 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:13:07 smithi195 ceph-mon[25840]: from='mgr.14806 172.21.15.89:0/3152262325' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T04:13:08.361 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:13:07 smithi195 ceph-mon[25840]: from='mgr.14806 172.21.15.89:0/3152262325' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-21T04:13:08.361 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:13:07 smithi195 ceph-mon[25840]: from='mgr.14806 172.21.15.89:0/3152262325' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T04:13:08.361 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:13:07 smithi195 ceph-mon[25840]: from='mgr.14806 172.21.15.89:0/3152262325' entity='mgr.smithi089.srlyxf' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi089.srlyxf/mirror_snapshot_schedule"}]: dispatch 2024-09-21T04:13:08.362 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:13:07 smithi195 ceph-mon[25840]: from='mgr.14806 172.21.15.89:0/3152262325' entity='mgr.smithi089.srlyxf' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi089.srlyxf/trash_purge_schedule"}]: dispatch 2024-09-21T04:13:08.362 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:13:07 smithi195 ceph-mon[25840]: mgrmap e33: smithi089.srlyxf(active, since 1.06847s), standbys: smithi195.cgotzm 2024-09-21T04:13:08.989 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:08 smithi089 ceph-mon[23733]: from='client.14814 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T04:13:08.989 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:08 smithi089 ceph-mon[23733]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-21T04:13:08.989 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:08 smithi089 ceph-mon[23733]: pgmap v3: 97 pgs: 97 active+clean; 324 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-21T04:13:09.043 INFO:teuthology.orchestra.run.smithi089.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-21T04:13:09.043 INFO:teuthology.orchestra.run.smithi089.stdout:alertmanager.smithi089 smithi089 *:9093,9094 running (80s) 17s ago 10m 17.3M - 0.27.0 11f11916f8cd 65dfcbcafee1 2024-09-21T04:13:09.043 INFO:teuthology.orchestra.run.smithi089.stdout:crash.smithi089 smithi089 running (10m) 17s ago 10m 6409k - 17.2.0 e1d6a67b021e d1e0a8969f78 2024-09-21T04:13:09.044 INFO:teuthology.orchestra.run.smithi089.stdout:crash.smithi195 smithi195 running (9m) 22s ago 9m 6400k - 17.2.0 e1d6a67b021e 7e213ff76084 2024-09-21T04:13:09.044 INFO:teuthology.orchestra.run.smithi089.stdout:grafana.smithi089 smithi089 *:3000 running (72s) 17s ago 10m 43.7M - dad864ee21e9 ec4d6738bce7 2024-09-21T04:13:09.044 INFO:teuthology.orchestra.run.smithi089.stdout:mds.foofs.smithi089.osufws smithi089 running (4m) 17s ago 4m 36.5M - 17.2.0 e1d6a67b021e 664d41859bea 2024-09-21T04:13:09.044 INFO:teuthology.orchestra.run.smithi089.stdout:mds.foofs.smithi195.hijxre smithi195 running (4m) 22s ago 4m 15.4M - 17.2.0 e1d6a67b021e 7bc0c6542d12 2024-09-21T04:13:09.044 INFO:teuthology.orchestra.run.smithi089.stdout:mgr.smithi089.srlyxf smithi089 *:8443,9283,8765 running (27s) 17s ago 11m 90.6M - 19.3.0-5103-gf36283e7 48ed0a2cf89d 3cbeaaa88241 2024-09-21T04:13:09.044 INFO:teuthology.orchestra.run.smithi089.stdout:mgr.smithi195.cgotzm smithi195 *:8443,9283 running (2m) 22s ago 9m 520M - 19.3.0-5103-gf36283e7 48ed0a2cf89d 18c1e9ebdeda 2024-09-21T04:13:09.044 INFO:teuthology.orchestra.run.smithi089.stdout:mon.smithi089 smithi089 running (11m) 17s ago 11m 52.3M 2048M 17.2.0 e1d6a67b021e 1d2b3719e46e 2024-09-21T04:13:09.044 INFO:teuthology.orchestra.run.smithi089.stdout:mon.smithi195 smithi195 running (9m) 22s ago 9m 42.7M 2048M 17.2.0 e1d6a67b021e 7f86c5b5d21e 2024-09-21T04:13:09.044 INFO:teuthology.orchestra.run.smithi089.stdout:nfs.foo.0.0.smithi089.dcjmqc smithi089 *:2049 running (4m) 17s ago 4m 260M - 4.0 e1d6a67b021e 98550f94e773 2024-09-21T04:13:09.044 INFO:teuthology.orchestra.run.smithi089.stdout:nfs.foo.1.0.smithi195.vhoayp smithi195 *:2049 running (3m) 22s ago 3m 83.3M - 4.0 e1d6a67b021e c60a4040130a 2024-09-21T04:13:09.044 INFO:teuthology.orchestra.run.smithi089.stdout:node-exporter.smithi089 smithi089 *:9100 running (64s) 17s ago 10m 10.0M - 1.7.0 72c9c2088986 7ef0d1588475 2024-09-21T04:13:09.044 INFO:teuthology.orchestra.run.smithi089.stdout:node-exporter.smithi195 smithi195 *:9100 running (35s) 22s ago 9m 7982k - 1.7.0 72c9c2088986 61eafd61bc6c 2024-09-21T04:13:09.044 INFO:teuthology.orchestra.run.smithi089.stdout:osd.0 smithi089 running (8m) 17s ago 8m 388M 1935M 17.2.0 e1d6a67b021e db7c0e5020e7 2024-09-21T04:13:09.044 INFO:teuthology.orchestra.run.smithi089.stdout:osd.1 smithi089 running (8m) 17s ago 8m 287M 1935M 17.2.0 e1d6a67b021e f9ec697d08d0 2024-09-21T04:13:09.044 INFO:teuthology.orchestra.run.smithi089.stdout:osd.2 smithi089 running (7m) 17s ago 7m 272M 1935M 17.2.0 e1d6a67b021e cfa000948859 2024-09-21T04:13:09.045 INFO:teuthology.orchestra.run.smithi089.stdout:osd.3 smithi089 running (7m) 17s ago 7m 358M 1935M 17.2.0 e1d6a67b021e 31c610ed5815 2024-09-21T04:13:09.045 INFO:teuthology.orchestra.run.smithi089.stdout:osd.4 smithi195 running (7m) 22s ago 7m 352M 2704M 17.2.0 e1d6a67b021e f00e18a1f220 2024-09-21T04:13:09.045 INFO:teuthology.orchestra.run.smithi089.stdout:osd.5 smithi195 running (6m) 22s ago 6m 386M 2704M 17.2.0 e1d6a67b021e cecfd6ff1eee 2024-09-21T04:13:09.045 INFO:teuthology.orchestra.run.smithi089.stdout:osd.6 smithi195 running (6m) 22s ago 6m 323M 2704M 17.2.0 e1d6a67b021e 0d7db362cc23 2024-09-21T04:13:09.045 INFO:teuthology.orchestra.run.smithi089.stdout:osd.7 smithi195 running (5m) 22s ago 5m 439M 2704M 17.2.0 e1d6a67b021e 0109b3fea188 2024-09-21T04:13:09.045 INFO:teuthology.orchestra.run.smithi089.stdout:prometheus.smithi089 smithi089 *:9095 running (42s) 17s ago 9m 39.6M - 2.51.0 1d3b7f56885b 9d48f60a4f6b 2024-09-21T04:13:09.318 INFO:teuthology.orchestra.run.smithi089.stdout: 5 107101 10.59 MB/sec execute 108 sec latency 52.973 ms 2024-09-21T04:13:09.358 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:13:08 smithi195 ceph-mon[25840]: from='client.14814 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T04:13:09.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:13:08 smithi195 ceph-mon[25840]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-21T04:13:09.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:13:08 smithi195 ceph-mon[25840]: pgmap v3: 97 pgs: 97 active+clean; 324 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-21T04:13:09.580 INFO:teuthology.orchestra.run.smithi089.stdout:{ 2024-09-21T04:13:09.580 INFO:teuthology.orchestra.run.smithi089.stdout: "mon": { 2024-09-21T04:13:09.580 INFO:teuthology.orchestra.run.smithi089.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-21T04:13:09.580 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2024-09-21T04:13:09.580 INFO:teuthology.orchestra.run.smithi089.stdout: "mgr": { 2024-09-21T04:13:09.580 INFO:teuthology.orchestra.run.smithi089.stdout: "ceph version 19.3.0-5103-gf36283e7 (f36283e7c91cac1c01cb9a7c4ff21bd697dbd4f2) squid (dev)": 2 2024-09-21T04:13:09.580 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2024-09-21T04:13:09.580 INFO:teuthology.orchestra.run.smithi089.stdout: "osd": { 2024-09-21T04:13:09.580 INFO:teuthology.orchestra.run.smithi089.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-09-21T04:13:09.581 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2024-09-21T04:13:09.581 INFO:teuthology.orchestra.run.smithi089.stdout: "mds": { 2024-09-21T04:13:09.581 INFO:teuthology.orchestra.run.smithi089.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-21T04:13:09.581 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2024-09-21T04:13:09.581 INFO:teuthology.orchestra.run.smithi089.stdout: "overall": { 2024-09-21T04:13:09.581 INFO:teuthology.orchestra.run.smithi089.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 12, 2024-09-21T04:13:09.581 INFO:teuthology.orchestra.run.smithi089.stdout: "ceph version 19.3.0-5103-gf36283e7 (f36283e7c91cac1c01cb9a7c4ff21bd697dbd4f2) squid (dev)": 2 2024-09-21T04:13:09.581 INFO:teuthology.orchestra.run.smithi089.stdout: } 2024-09-21T04:13:09.581 INFO:teuthology.orchestra.run.smithi089.stdout:} 2024-09-21T04:13:10.015 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:09 smithi089 ceph-mon[23733]: [21/Sep/2024:04:13:08] ENGINE Bus STARTING 2024-09-21T04:13:10.015 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:09 smithi089 ceph-mon[23733]: [21/Sep/2024:04:13:08] ENGINE Serving on https://172.21.15.89:7150 2024-09-21T04:13:10.015 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:09 smithi089 ceph-mon[23733]: [21/Sep/2024:04:13:08] ENGINE Error in HTTPServer.serve 2024-09-21T04:13:10.015 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout: Traceback (most recent call last): 2024-09-21T04:13:10.015 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-21T04:13:10.015 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout: self._connections.run(self.expiration_interval) 2024-09-21T04:13:10.015 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-21T04:13:10.016 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout: self._run(expiration_interval) 2024-09-21T04:13:10.016 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-21T04:13:10.016 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-21T04:13:10.016 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-21T04:13:10.016 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-21T04:13:10.016 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-21T04:13:10.016 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout: s = self.context.wrap_socket( 2024-09-21T04:13:10.016 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-21T04:13:10.016 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout: return self.sslsocket_class._create( 2024-09-21T04:13:10.016 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-21T04:13:10.016 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout: self.do_handshake() 2024-09-21T04:13:10.017 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-21T04:13:10.017 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout: self._sslobj.do_handshake() 2024-09-21T04:13:10.017 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-21T04:13:10.017 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:09 smithi089 ceph-mon[23733]: pgmap v4: 97 pgs: 97 active+clean; 324 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-21T04:13:10.017 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:09 smithi089 ceph-mon[23733]: [21/Sep/2024:04:13:08] ENGINE Serving on http://172.21.15.89:8765 2024-09-21T04:13:10.017 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:09 smithi089 ceph-mon[23733]: [21/Sep/2024:04:13:08] ENGINE Bus STARTED 2024-09-21T04:13:10.017 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:09 smithi089 ceph-mon[23733]: mgrmap e34: smithi089.srlyxf(active, since 2s), standbys: smithi195.cgotzm 2024-09-21T04:13:10.017 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:09 smithi089 ceph-mon[23733]: from='client.14832 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T04:13:10.017 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:09 smithi089 ceph-mon[23733]: from='client.? 172.21.15.89:0/3185949568' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T04:13:10.035 INFO:teuthology.orchestra.run.smithi089.stdout:{ 2024-09-21T04:13:10.036 INFO:teuthology.orchestra.run.smithi089.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:2195e5f178be0a9390728e2a14088236d8fe6f82635f5c84418fe37a04457e00", 2024-09-21T04:13:10.036 INFO:teuthology.orchestra.run.smithi089.stdout: "in_progress": true, 2024-09-21T04:13:10.036 INFO:teuthology.orchestra.run.smithi089.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-21T04:13:10.036 INFO:teuthology.orchestra.run.smithi089.stdout: "services_complete": [ 2024-09-21T04:13:10.036 INFO:teuthology.orchestra.run.smithi089.stdout: "mgr" 2024-09-21T04:13:10.036 INFO:teuthology.orchestra.run.smithi089.stdout: ], 2024-09-21T04:13:10.036 INFO:teuthology.orchestra.run.smithi089.stdout: "progress": "2/23 daemons upgraded", 2024-09-21T04:13:10.036 INFO:teuthology.orchestra.run.smithi089.stdout: "message": "", 2024-09-21T04:13:10.036 INFO:teuthology.orchestra.run.smithi089.stdout: "is_paused": false 2024-09-21T04:13:10.036 INFO:teuthology.orchestra.run.smithi089.stdout:} 2024-09-21T04:13:10.319 INFO:teuthology.orchestra.run.smithi089.stdout: 5 107380 10.53 MB/sec execute 109 sec latency 39.709 ms 2024-09-21T04:13:10.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:13:09 smithi195 ceph-mon[25840]: [21/Sep/2024:04:13:08] ENGINE Bus STARTING 2024-09-21T04:13:10.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:13:09 smithi195 ceph-mon[25840]: [21/Sep/2024:04:13:08] ENGINE Serving on https://172.21.15.89:7150 2024-09-21T04:13:10.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:13:09 smithi195 ceph-mon[25840]: [21/Sep/2024:04:13:08] ENGINE Error in HTTPServer.serve 2024-09-21T04:13:10.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout: Traceback (most recent call last): 2024-09-21T04:13:10.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-21T04:13:10.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout: self._connections.run(self.expiration_interval) 2024-09-21T04:13:10.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-21T04:13:10.360 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout: self._run(expiration_interval) 2024-09-21T04:13:10.360 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-21T04:13:10.360 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-21T04:13:10.360 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-21T04:13:10.360 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-21T04:13:10.360 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-21T04:13:10.360 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout: s = self.context.wrap_socket( 2024-09-21T04:13:10.360 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-21T04:13:10.360 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout: return self.sslsocket_class._create( 2024-09-21T04:13:10.361 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-21T04:13:10.361 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout: self.do_handshake() 2024-09-21T04:13:10.361 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-21T04:13:10.361 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout: self._sslobj.do_handshake() 2024-09-21T04:13:10.361 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-21T04:13:10.361 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:13:09 smithi195 ceph-mon[25840]: pgmap v4: 97 pgs: 97 active+clean; 324 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-21T04:13:10.361 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:13:09 smithi195 ceph-mon[25840]: [21/Sep/2024:04:13:08] ENGINE Serving on http://172.21.15.89:8765 2024-09-21T04:13:10.361 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:13:09 smithi195 ceph-mon[25840]: [21/Sep/2024:04:13:08] ENGINE Bus STARTED 2024-09-21T04:13:10.361 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:13:09 smithi195 ceph-mon[25840]: mgrmap e34: smithi089.srlyxf(active, since 2s), standbys: smithi195.cgotzm 2024-09-21T04:13:10.361 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:13:09 smithi195 ceph-mon[25840]: from='client.14832 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T04:13:10.362 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:13:09 smithi195 ceph-mon[25840]: from='client.? 172.21.15.89:0/3185949568' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T04:13:10.632 INFO:teuthology.orchestra.run.smithi089.stdout:HEALTH_OK 2024-09-21T04:13:11.047 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:10 smithi089 ceph-mon[23733]: from='client.24551 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T04:13:11.048 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:10 smithi089 ceph-mon[23733]: from='client.? 172.21.15.89:0/2026776075' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-21T04:13:11.108 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:13:11 smithi195 ceph-mon[25840]: from='client.24551 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T04:13:11.109 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:13:11 smithi195 ceph-mon[25840]: from='client.? 172.21.15.89:0/2026776075' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-21T04:13:11.318 INFO:teuthology.orchestra.run.smithi089.stdout: 5 107706 10.53 MB/sec execute 110 sec latency 103.382 ms 2024-09-21T04:13:12.077 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:11 smithi089 ceph-mon[23733]: pgmap v5: 97 pgs: 97 active+clean; 324 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-21T04:13:12.078 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:11 smithi089 ceph-mon[23733]: mgrmap e35: smithi089.srlyxf(active, since 4s), standbys: smithi195.cgotzm 2024-09-21T04:13:12.319 INFO:teuthology.orchestra.run.smithi089.stdout: 5 108179 10.53 MB/sec execute 111 sec latency 29.099 ms 2024-09-21T04:13:12.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:13:11 smithi195 ceph-mon[25840]: pgmap v5: 97 pgs: 97 active+clean; 324 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-21T04:13:12.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:13:11 smithi195 ceph-mon[25840]: mgrmap e35: smithi089.srlyxf(active, since 4s), standbys: smithi195.cgotzm 2024-09-21T04:13:13.319 INFO:teuthology.orchestra.run.smithi089.stdout: 5 108694 10.49 MB/sec execute 112 sec latency 32.978 ms 2024-09-21T04:13:13.597 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:13 smithi089 ceph-mon[23733]: from='mgr.14806 172.21.15.89:0/3152262325' entity='mgr.smithi089.srlyxf' 2024-09-21T04:13:13.598 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:13 smithi089 ceph-mon[23733]: from='mgr.14806 172.21.15.89:0/3152262325' entity='mgr.smithi089.srlyxf' 2024-09-21T04:13:13.598 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:13 smithi089 ceph-mon[23733]: pgmap v6: 97 pgs: 97 active+clean; 324 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-21T04:13:13.598 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:13 smithi089 ceph-mon[23733]: from='mgr.14806 172.21.15.89:0/3152262325' entity='mgr.smithi089.srlyxf' 2024-09-21T04:13:13.598 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:13 smithi089 ceph-mon[23733]: from='mgr.14806 172.21.15.89:0/3152262325' entity='mgr.smithi089.srlyxf' 2024-09-21T04:13:13.608 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:13:13 smithi195 ceph-mon[25840]: from='mgr.14806 172.21.15.89:0/3152262325' entity='mgr.smithi089.srlyxf' 2024-09-21T04:13:13.608 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:13:13 smithi195 ceph-mon[25840]: from='mgr.14806 172.21.15.89:0/3152262325' entity='mgr.smithi089.srlyxf' 2024-09-21T04:13:13.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:13:13 smithi195 ceph-mon[25840]: pgmap v6: 97 pgs: 97 active+clean; 324 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-21T04:13:13.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:13:13 smithi195 ceph-mon[25840]: from='mgr.14806 172.21.15.89:0/3152262325' entity='mgr.smithi089.srlyxf' 2024-09-21T04:13:13.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:13:13 smithi195 ceph-mon[25840]: from='mgr.14806 172.21.15.89:0/3152262325' entity='mgr.smithi089.srlyxf' 2024-09-21T04:13:14.319 INFO:teuthology.orchestra.run.smithi089.stdout: 5 109010 10.48 MB/sec execute 113 sec latency 54.101 ms 2024-09-21T04:13:15.319 INFO:teuthology.orchestra.run.smithi089.stdout: 5 109400 10.53 MB/sec execute 114 sec latency 75.565 ms 2024-09-21T04:13:16.179 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:16 smithi089 ceph-mon[23733]: pgmap v7: 97 pgs: 97 active+clean; 312 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 33 KiB/s rd, 2.9 MiB/s wr, 395 op/s 2024-09-21T04:13:16.179 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:16 smithi089 ceph-mon[23733]: from='mgr.14806 172.21.15.89:0/3152262325' entity='mgr.smithi089.srlyxf' 2024-09-21T04:13:16.319 INFO:teuthology.orchestra.run.smithi089.stdout: 5 109810 10.58 MB/sec execute 115 sec latency 36.779 ms 2024-09-21T04:13:16.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:13:16 smithi195 ceph-mon[25840]: pgmap v7: 97 pgs: 97 active+clean; 312 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 33 KiB/s rd, 2.9 MiB/s wr, 395 op/s 2024-09-21T04:13:16.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:13:16 smithi195 ceph-mon[25840]: from='mgr.14806 172.21.15.89:0/3152262325' entity='mgr.smithi089.srlyxf' 2024-09-21T04:13:17.205 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:17 smithi089 ceph-mon[23733]: from='mgr.14806 172.21.15.89:0/3152262325' entity='mgr.smithi089.srlyxf' 2024-09-21T04:13:17.205 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:17 smithi089 ceph-mon[23733]: from='mgr.14806 172.21.15.89:0/3152262325' entity='mgr.smithi089.srlyxf' 2024-09-21T04:13:17.205 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:17 smithi089 ceph-mon[23733]: from='mgr.14806 172.21.15.89:0/3152262325' entity='mgr.smithi089.srlyxf' 2024-09-21T04:13:17.319 INFO:teuthology.orchestra.run.smithi089.stdout: 5 110139 10.63 MB/sec execute 116 sec latency 41.696 ms 2024-09-21T04:13:17.358 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:13:17 smithi195 ceph-mon[25840]: from='mgr.14806 172.21.15.89:0/3152262325' entity='mgr.smithi089.srlyxf' 2024-09-21T04:13:17.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:13:17 smithi195 ceph-mon[25840]: from='mgr.14806 172.21.15.89:0/3152262325' entity='mgr.smithi089.srlyxf' 2024-09-21T04:13:17.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:13:17 smithi195 ceph-mon[25840]: from='mgr.14806 172.21.15.89:0/3152262325' entity='mgr.smithi089.srlyxf' 2024-09-21T04:13:18.230 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:18 smithi089 ceph-mon[23733]: pgmap v8: 97 pgs: 97 active+clean; 312 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 25 KiB/s rd, 2.2 MiB/s wr, 307 op/s 2024-09-21T04:13:18.319 INFO:teuthology.orchestra.run.smithi089.stdout: 5 110341 10.57 MB/sec execute 117 sec latency 48.587 ms 2024-09-21T04:13:18.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:13:18 smithi195 ceph-mon[25840]: pgmap v8: 97 pgs: 97 active+clean; 312 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 25 KiB/s rd, 2.2 MiB/s wr, 307 op/s 2024-09-21T04:13:19.319 INFO:teuthology.orchestra.run.smithi089.stdout: 5 110559 10.50 MB/sec execute 118 sec latency 61.366 ms 2024-09-21T04:13:20.271 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:20 smithi089 ceph-mon[23733]: pgmap v9: 97 pgs: 97 active+clean; 324 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 23 KiB/s rd, 4.7 MiB/s wr, 470 op/s 2024-09-21T04:13:20.319 INFO:teuthology.orchestra.run.smithi089.stdout: 5 110830 10.43 MB/sec execute 119 sec latency 105.776 ms 2024-09-21T04:13:20.358 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:13:20 smithi195 ceph-mon[25840]: pgmap v9: 97 pgs: 97 active+clean; 324 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 23 KiB/s rd, 4.7 MiB/s wr, 470 op/s 2024-09-21T04:13:21.319 INFO:teuthology.orchestra.run.smithi089.stdout: 5 111107 10.41 MB/sec execute 120 sec latency 69.123 ms 2024-09-21T04:13:22.068 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:22 smithi089 ceph-mon[23733]: pgmap v10: 97 pgs: 97 active+clean; 324 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 21 KiB/s rd, 4.3 MiB/s wr, 429 op/s 2024-09-21T04:13:22.319 INFO:teuthology.orchestra.run.smithi089.stdout: 5 111480 10.40 MB/sec execute 121 sec latency 56.598 ms 2024-09-21T04:13:22.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:13:22 smithi195 ceph-mon[25840]: pgmap v10: 97 pgs: 97 active+clean; 324 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 21 KiB/s rd, 4.3 MiB/s wr, 429 op/s 2024-09-21T04:13:23.321 INFO:teuthology.orchestra.run.smithi089.stdout: 5 111859 10.38 MB/sec execute 122 sec latency 79.779 ms 2024-09-21T04:13:23.572 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:23 smithi089 ceph-mon[23733]: from='mgr.14806 172.21.15.89:0/3152262325' entity='mgr.smithi089.srlyxf' 2024-09-21T04:13:23.572 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:23 smithi089 ceph-mon[23733]: from='mgr.14806 172.21.15.89:0/3152262325' entity='mgr.smithi089.srlyxf' 2024-09-21T04:13:23.572 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:23 smithi089 ceph-mon[23733]: from='mgr.14806 172.21.15.89:0/3152262325' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T04:13:23.572 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:23 smithi089 ceph-mon[23733]: from='mgr.14806 172.21.15.89:0/3152262325' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T04:13:23.572 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:23 smithi089 ceph-mon[23733]: Updating smithi089:/etc/ceph/ceph.conf 2024-09-21T04:13:23.572 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:23 smithi089 ceph-mon[23733]: Updating smithi195:/etc/ceph/ceph.conf 2024-09-21T04:13:23.572 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:23 smithi089 ceph-mon[23733]: from='mgr.14806 172.21.15.89:0/3152262325' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T04:13:23.572 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:23 smithi089 ceph-mon[23733]: Updating smithi195:/var/lib/ceph/1b9b79fc-77ce-11ef-baf5-efdc52797490/config/ceph.conf 2024-09-21T04:13:23.572 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:23 smithi089 ceph-mon[23733]: Updating smithi089:/var/lib/ceph/1b9b79fc-77ce-11ef-baf5-efdc52797490/config/ceph.conf 2024-09-21T04:13:23.573 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:23 smithi089 ceph-mon[23733]: Updating smithi195:/etc/ceph/ceph.client.admin.keyring 2024-09-21T04:13:23.573 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:23 smithi089 ceph-mon[23733]: Updating smithi089:/etc/ceph/ceph.client.admin.keyring 2024-09-21T04:13:23.573 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:23 smithi089 ceph-mon[23733]: Updating smithi195:/var/lib/ceph/1b9b79fc-77ce-11ef-baf5-efdc52797490/config/ceph.client.admin.keyring 2024-09-21T04:13:23.573 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:23 smithi089 ceph-mon[23733]: pgmap v11: 97 pgs: 97 active+clean; 324 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 21 KiB/s rd, 4.3 MiB/s wr, 429 op/s 2024-09-21T04:13:23.573 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:23 smithi089 ceph-mon[23733]: Updating smithi089:/var/lib/ceph/1b9b79fc-77ce-11ef-baf5-efdc52797490/config/ceph.client.admin.keyring 2024-09-21T04:13:23.573 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:23 smithi089 ceph-mon[23733]: from='mgr.14806 172.21.15.89:0/3152262325' entity='mgr.smithi089.srlyxf' 2024-09-21T04:13:23.573 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:23 smithi089 ceph-mon[23733]: from='mgr.14806 172.21.15.89:0/3152262325' entity='mgr.smithi089.srlyxf' 2024-09-21T04:13:23.573 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:23 smithi089 ceph-mon[23733]: from='mgr.14806 172.21.15.89:0/3152262325' entity='mgr.smithi089.srlyxf' 2024-09-21T04:13:23.573 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:23 smithi089 ceph-mon[23733]: from='mgr.14806 172.21.15.89:0/3152262325' entity='mgr.smithi089.srlyxf' 2024-09-21T04:13:23.573 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:23 smithi089 ceph-mon[23733]: from='mgr.14806 172.21.15.89:0/3152262325' entity='mgr.smithi089.srlyxf' 2024-09-21T04:13:23.573 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:23 smithi089 ceph-mon[23733]: from='mgr.14806 172.21.15.89:0/3152262325' entity='mgr.smithi089.srlyxf' 2024-09-21T04:13:23.859 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:13:23 smithi195 ceph-mon[25840]: from='mgr.14806 172.21.15.89:0/3152262325' entity='mgr.smithi089.srlyxf' 2024-09-21T04:13:23.859 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:13:23 smithi195 ceph-mon[25840]: from='mgr.14806 172.21.15.89:0/3152262325' entity='mgr.smithi089.srlyxf' 2024-09-21T04:13:23.859 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:13:23 smithi195 ceph-mon[25840]: from='mgr.14806 172.21.15.89:0/3152262325' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T04:13:23.859 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:13:23 smithi195 ceph-mon[25840]: from='mgr.14806 172.21.15.89:0/3152262325' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T04:13:23.859 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:13:23 smithi195 ceph-mon[25840]: Updating smithi089:/etc/ceph/ceph.conf 2024-09-21T04:13:23.859 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:13:23 smithi195 ceph-mon[25840]: Updating smithi195:/etc/ceph/ceph.conf 2024-09-21T04:13:23.859 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:13:23 smithi195 ceph-mon[25840]: from='mgr.14806 172.21.15.89:0/3152262325' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T04:13:23.859 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:13:23 smithi195 ceph-mon[25840]: Updating smithi195:/var/lib/ceph/1b9b79fc-77ce-11ef-baf5-efdc52797490/config/ceph.conf 2024-09-21T04:13:23.859 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:13:23 smithi195 ceph-mon[25840]: Updating smithi089:/var/lib/ceph/1b9b79fc-77ce-11ef-baf5-efdc52797490/config/ceph.conf 2024-09-21T04:13:23.859 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:13:23 smithi195 ceph-mon[25840]: Updating smithi195:/etc/ceph/ceph.client.admin.keyring 2024-09-21T04:13:23.859 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:13:23 smithi195 ceph-mon[25840]: Updating smithi089:/etc/ceph/ceph.client.admin.keyring 2024-09-21T04:13:23.860 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:13:23 smithi195 ceph-mon[25840]: Updating smithi195:/var/lib/ceph/1b9b79fc-77ce-11ef-baf5-efdc52797490/config/ceph.client.admin.keyring 2024-09-21T04:13:23.860 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:13:23 smithi195 ceph-mon[25840]: pgmap v11: 97 pgs: 97 active+clean; 324 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 21 KiB/s rd, 4.3 MiB/s wr, 429 op/s 2024-09-21T04:13:23.860 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:13:23 smithi195 ceph-mon[25840]: Updating smithi089:/var/lib/ceph/1b9b79fc-77ce-11ef-baf5-efdc52797490/config/ceph.client.admin.keyring 2024-09-21T04:13:23.860 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:13:23 smithi195 ceph-mon[25840]: from='mgr.14806 172.21.15.89:0/3152262325' entity='mgr.smithi089.srlyxf' 2024-09-21T04:13:23.860 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:13:23 smithi195 ceph-mon[25840]: from='mgr.14806 172.21.15.89:0/3152262325' entity='mgr.smithi089.srlyxf' 2024-09-21T04:13:23.860 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:13:23 smithi195 ceph-mon[25840]: from='mgr.14806 172.21.15.89:0/3152262325' entity='mgr.smithi089.srlyxf' 2024-09-21T04:13:23.860 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:13:23 smithi195 ceph-mon[25840]: from='mgr.14806 172.21.15.89:0/3152262325' entity='mgr.smithi089.srlyxf' 2024-09-21T04:13:23.860 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:13:23 smithi195 ceph-mon[25840]: from='mgr.14806 172.21.15.89:0/3152262325' entity='mgr.smithi089.srlyxf' 2024-09-21T04:13:23.860 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:13:23 smithi195 ceph-mon[25840]: from='mgr.14806 172.21.15.89:0/3152262325' entity='mgr.smithi089.srlyxf' 2024-09-21T04:13:24.321 INFO:teuthology.orchestra.run.smithi089.stdout: 5 112360 10.34 MB/sec execute 123 sec latency 37.581 ms 2024-09-21T04:13:24.586 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:24 smithi089 ceph-mon[23733]: Reconfiguring prometheus.smithi089 (dependencies changed)... 2024-09-21T04:13:24.586 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:24 smithi089 ceph-mon[23733]: Reconfiguring daemon prometheus.smithi089 on smithi089 2024-09-21T04:13:24.859 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:13:24 smithi195 ceph-mon[25840]: Reconfiguring prometheus.smithi089 (dependencies changed)... 2024-09-21T04:13:24.859 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:13:24 smithi195 ceph-mon[25840]: Reconfiguring daemon prometheus.smithi089 on smithi089 2024-09-21T04:13:25.322 INFO:teuthology.orchestra.run.smithi089.stdout: 5 112598 10.33 MB/sec execute 124 sec latency 58.236 ms 2024-09-21T04:13:25.647 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:25 smithi089 ceph-mon[23733]: pgmap v12: 97 pgs: 97 active+clean; 316 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 25 KiB/s rd, 5.8 MiB/s wr, 627 op/s 2024-09-21T04:13:25.858 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:13:25 smithi195 ceph-mon[25840]: pgmap v12: 97 pgs: 97 active+clean; 316 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 25 KiB/s rd, 5.8 MiB/s wr, 627 op/s 2024-09-21T04:13:26.322 INFO:teuthology.orchestra.run.smithi089.stdout: 5 113039 10.43 MB/sec execute 125 sec latency 53.337 ms 2024-09-21T04:13:27.322 INFO:teuthology.orchestra.run.smithi089.stdout: 5 113417 10.43 MB/sec execute 126 sec latency 34.403 ms 2024-09-21T04:13:28.245 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:28 smithi089 ceph-mon[23733]: pgmap v13: 97 pgs: 97 active+clean; 316 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.2 MiB/s wr, 398 op/s 2024-09-21T04:13:28.322 INFO:teuthology.orchestra.run.smithi089.stdout: 5 113737 10.47 MB/sec execute 127 sec latency 71.751 ms 2024-09-21T04:13:28.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:13:28 smithi195 ceph-mon[25840]: pgmap v13: 97 pgs: 97 active+clean; 316 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.2 MiB/s wr, 398 op/s 2024-09-21T04:13:29.322 INFO:teuthology.orchestra.run.smithi089.stdout: 5 113927 10.41 MB/sec execute 128 sec latency 71.353 ms 2024-09-21T04:13:30.027 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:30 smithi089 ceph-mon[23733]: pgmap v14: 97 pgs: 97 active+clean; 324 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 8.5 KiB/s rd, 6.6 MiB/s wr, 606 op/s 2024-09-21T04:13:30.324 INFO:teuthology.orchestra.run.smithi089.stdout: 5 114095 10.35 MB/sec execute 129 sec latency 66.634 ms 2024-09-21T04:13:30.358 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:13:30 smithi195 ceph-mon[25840]: pgmap v14: 97 pgs: 97 active+clean; 324 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 8.5 KiB/s rd, 6.6 MiB/s wr, 606 op/s 2024-09-21T04:13:31.324 INFO:teuthology.orchestra.run.smithi089.stdout: 5 114437 10.29 MB/sec execute 130 sec latency 59.603 ms 2024-09-21T04:13:32.105 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:32 smithi089 ceph-mon[23733]: pgmap v15: 97 pgs: 97 active+clean; 324 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.0 MiB/s wr, 406 op/s 2024-09-21T04:13:32.106 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:32 smithi089 ceph-mon[23733]: from='mgr.14806 172.21.15.89:0/3152262325' entity='mgr.smithi089.srlyxf' 2024-09-21T04:13:32.106 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:32 smithi089 ceph-mon[23733]: from='mgr.14806 172.21.15.89:0/3152262325' entity='mgr.smithi089.srlyxf' 2024-09-21T04:13:32.106 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:32 smithi089 ceph-mon[23733]: from='mgr.14806 172.21.15.89:0/3152262325' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-21T04:13:32.106 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:32 smithi089 ceph-mon[23733]: from='mgr.14806 172.21.15.89:0/3152262325' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T04:13:32.287 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:13:32 smithi195 ceph-mon[25840]: pgmap v15: 97 pgs: 97 active+clean; 324 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.0 MiB/s wr, 406 op/s 2024-09-21T04:13:32.288 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:13:32 smithi195 ceph-mon[25840]: from='mgr.14806 172.21.15.89:0/3152262325' entity='mgr.smithi089.srlyxf' 2024-09-21T04:13:32.288 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:13:32 smithi195 ceph-mon[25840]: from='mgr.14806 172.21.15.89:0/3152262325' entity='mgr.smithi089.srlyxf' 2024-09-21T04:13:32.288 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:13:32 smithi195 ceph-mon[25840]: from='mgr.14806 172.21.15.89:0/3152262325' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-21T04:13:32.288 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:13:32 smithi195 ceph-mon[25840]: from='mgr.14806 172.21.15.89:0/3152262325' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T04:13:32.324 INFO:teuthology.orchestra.run.smithi089.stdout: 5 114772 10.28 MB/sec execute 131 sec latency 43.339 ms 2024-09-21T04:13:33.111 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:33 smithi089 ceph-mon[23733]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-21T04:13:33.324 INFO:teuthology.orchestra.run.smithi089.stdout: 5 115604 10.33 MB/sec execute 132 sec latency 29.552 ms 2024-09-21T04:13:33.358 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:13:33 smithi195 ceph-mon[25840]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-21T04:13:34.109 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:13:34 smithi195 ceph-mon[25840]: pgmap v16: 97 pgs: 97 active+clean; 324 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.0 MiB/s wr, 406 op/s 2024-09-21T04:13:34.109 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:13:34 smithi195 ceph-mon[25840]: from='mgr.14806 172.21.15.89:0/3152262325' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi195.cgotzm", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-21T04:13:34.109 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:13:34 smithi195 ceph-mon[25840]: from='mgr.14806 172.21.15.89:0/3152262325' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-21T04:13:34.109 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:13:34 smithi195 ceph-mon[25840]: from='mgr.14806 172.21.15.89:0/3152262325' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T04:13:34.142 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:34 smithi089 ceph-mon[23733]: pgmap v16: 97 pgs: 97 active+clean; 324 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.0 MiB/s wr, 406 op/s 2024-09-21T04:13:34.143 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:34 smithi089 ceph-mon[23733]: from='mgr.14806 172.21.15.89:0/3152262325' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi195.cgotzm", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-21T04:13:34.143 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:34 smithi089 ceph-mon[23733]: from='mgr.14806 172.21.15.89:0/3152262325' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-21T04:13:34.143 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:34 smithi089 ceph-mon[23733]: from='mgr.14806 172.21.15.89:0/3152262325' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T04:13:34.327 INFO:teuthology.orchestra.run.smithi089.stdout: 5 116117 10.35 MB/sec execute 133 sec latency 43.356 ms 2024-09-21T04:13:35.161 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:35 smithi089 ceph-mon[23733]: Upgrade: Updating mgr.smithi195.cgotzm 2024-09-21T04:13:35.161 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:35 smithi089 ceph-mon[23733]: Deploying daemon mgr.smithi195.cgotzm on smithi195 2024-09-21T04:13:35.327 INFO:teuthology.orchestra.run.smithi089.stdout: 5 116683 10.46 MB/sec execute 134 sec latency 34.083 ms 2024-09-21T04:13:35.453 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:13:35 smithi195 ceph-mon[25840]: Upgrade: Updating mgr.smithi195.cgotzm 2024-09-21T04:13:35.453 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:13:35 smithi195 ceph-mon[25840]: Deploying daemon mgr.smithi195.cgotzm on smithi195 2024-09-21T04:13:36.168 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:36 smithi089 ceph-mon[23733]: pgmap v17: 97 pgs: 97 active+clean; 319 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 5.9 MiB/s wr, 627 op/s 2024-09-21T04:13:36.328 INFO:teuthology.orchestra.run.smithi089.stdout: 5 117101 10.49 MB/sec execute 135 sec latency 50.624 ms 2024-09-21T04:13:36.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:13:36 smithi195 ceph-mon[25840]: pgmap v17: 97 pgs: 97 active+clean; 319 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 5.9 MiB/s wr, 627 op/s 2024-09-21T04:13:37.327 INFO:teuthology.orchestra.run.smithi089.stdout: 5 117330 10.49 MB/sec execute 136 sec latency 78.697 ms 2024-09-21T04:13:38.241 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:38 smithi089 ceph-mon[23733]: pgmap v18: 97 pgs: 97 active+clean; 319 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.4 MiB/s wr, 428 op/s 2024-09-21T04:13:38.241 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:38 smithi089 ceph-mon[23733]: from='mgr.14806 172.21.15.89:0/3152262325' entity='mgr.smithi089.srlyxf' 2024-09-21T04:13:38.241 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:38 smithi089 ceph-mon[23733]: from='mgr.14806 172.21.15.89:0/3152262325' entity='mgr.smithi089.srlyxf' 2024-09-21T04:13:38.241 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:38 smithi089 ceph-mon[23733]: from='mgr.14806 172.21.15.89:0/3152262325' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T04:13:38.241 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:38 smithi089 ceph-mon[23733]: from='mgr.14806 172.21.15.89:0/3152262325' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T04:13:38.241 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:38 smithi089 ceph-mon[23733]: from='mgr.14806 172.21.15.89:0/3152262325' entity='mgr.smithi089.srlyxf' 2024-09-21T04:13:38.241 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:38 smithi089 ceph-mon[23733]: from='mgr.14806 172.21.15.89:0/3152262325' entity='mgr.smithi089.srlyxf' 2024-09-21T04:13:38.327 INFO:teuthology.orchestra.run.smithi089.stdout: 5 117511 10.43 MB/sec execute 137 sec latency 45.108 ms 2024-09-21T04:13:38.341 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:13:38 smithi195 ceph-mon[25840]: pgmap v18: 97 pgs: 97 active+clean; 319 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.4 MiB/s wr, 428 op/s 2024-09-21T04:13:38.342 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:13:38 smithi195 ceph-mon[25840]: from='mgr.14806 172.21.15.89:0/3152262325' entity='mgr.smithi089.srlyxf' 2024-09-21T04:13:38.342 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:13:38 smithi195 ceph-mon[25840]: from='mgr.14806 172.21.15.89:0/3152262325' entity='mgr.smithi089.srlyxf' 2024-09-21T04:13:38.342 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:13:38 smithi195 ceph-mon[25840]: from='mgr.14806 172.21.15.89:0/3152262325' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T04:13:38.342 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:13:38 smithi195 ceph-mon[25840]: from='mgr.14806 172.21.15.89:0/3152262325' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T04:13:38.342 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:13:38 smithi195 ceph-mon[25840]: from='mgr.14806 172.21.15.89:0/3152262325' entity='mgr.smithi089.srlyxf' 2024-09-21T04:13:38.342 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:13:38 smithi195 ceph-mon[25840]: from='mgr.14806 172.21.15.89:0/3152262325' entity='mgr.smithi089.srlyxf' 2024-09-21T04:13:39.328 INFO:teuthology.orchestra.run.smithi089.stdout: 5 117735 10.37 MB/sec execute 138 sec latency 112.353 ms 2024-09-21T04:13:40.295 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:40 smithi089 ceph-mon[23733]: pgmap v19: 97 pgs: 97 active+clean; 327 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 9.2 KiB/s rd, 7.4 MiB/s wr, 673 op/s 2024-09-21T04:13:40.327 INFO:teuthology.orchestra.run.smithi089.stdout: 5 117996 10.31 MB/sec execute 139 sec latency 69.555 ms 2024-09-21T04:13:40.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:13:40 smithi195 ceph-mon[25840]: pgmap v19: 97 pgs: 97 active+clean; 327 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 9.2 KiB/s rd, 7.4 MiB/s wr, 673 op/s 2024-09-21T04:13:41.109 INFO:teuthology.orchestra.run.smithi089.stdout:true 2024-09-21T04:13:41.328 INFO:teuthology.orchestra.run.smithi089.stdout: 5 118254 10.29 MB/sec execute 140 sec latency 58.312 ms 2024-09-21T04:13:42.047 INFO:teuthology.orchestra.run.smithi089.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-21T04:13:42.048 INFO:teuthology.orchestra.run.smithi089.stdout:alertmanager.smithi089 smithi089 *:9093,9094 running (113s) 26s ago 10m 19.5M - 0.27.0 11f11916f8cd 65dfcbcafee1 2024-09-21T04:13:42.048 INFO:teuthology.orchestra.run.smithi089.stdout:crash.smithi089 smithi089 running (10m) 26s ago 10m 6409k - 17.2.0 e1d6a67b021e d1e0a8969f78 2024-09-21T04:13:42.048 INFO:teuthology.orchestra.run.smithi089.stdout:crash.smithi195 smithi195 running (10m) 0s ago 10m 6400k - 17.2.0 e1d6a67b021e 7e213ff76084 2024-09-21T04:13:42.048 INFO:teuthology.orchestra.run.smithi089.stdout:grafana.smithi089 smithi089 *:3000 running (105s) 26s ago 10m 44.8M - dad864ee21e9 ec4d6738bce7 2024-09-21T04:13:42.048 INFO:teuthology.orchestra.run.smithi089.stdout:mds.foofs.smithi089.osufws smithi089 running (4m) 26s ago 4m 37.6M - 17.2.0 e1d6a67b021e 664d41859bea 2024-09-21T04:13:42.048 INFO:teuthology.orchestra.run.smithi089.stdout:mds.foofs.smithi195.hijxre smithi195 running (4m) 0s ago 4m 16.0M - 17.2.0 e1d6a67b021e 7bc0c6542d12 2024-09-21T04:13:42.048 INFO:teuthology.orchestra.run.smithi089.stdout:mgr.smithi089.srlyxf smithi089 *:8443,9283,8765 running (60s) 26s ago 12m 489M - 19.3.0-5103-gf36283e7 48ed0a2cf89d 3cbeaaa88241 2024-09-21T04:13:42.048 INFO:teuthology.orchestra.run.smithi089.stdout:mgr.smithi195.cgotzm smithi195 *:8443,9283,8765 running (5s) 0s ago 10m 88.3M - 19.3.0-5103-gf36283e7 48ed0a2cf89d d3734574cf66 2024-09-21T04:13:42.048 INFO:teuthology.orchestra.run.smithi089.stdout:mon.smithi089 smithi089 running (12m) 26s ago 12m 56.1M 2048M 17.2.0 e1d6a67b021e 1d2b3719e46e 2024-09-21T04:13:42.048 INFO:teuthology.orchestra.run.smithi089.stdout:mon.smithi195 smithi195 running (10m) 0s ago 10m 43.8M 2048M 17.2.0 e1d6a67b021e 7f86c5b5d21e 2024-09-21T04:13:42.048 INFO:teuthology.orchestra.run.smithi089.stdout:nfs.foo.0.0.smithi089.dcjmqc smithi089 *:2049 running (4m) 26s ago 4m 267M - 4.0 e1d6a67b021e 98550f94e773 2024-09-21T04:13:42.048 INFO:teuthology.orchestra.run.smithi089.stdout:nfs.foo.1.0.smithi195.vhoayp smithi195 *:2049 running (4m) 0s ago 4m 87.2M - 4.0 e1d6a67b021e c60a4040130a 2024-09-21T04:13:42.048 INFO:teuthology.orchestra.run.smithi089.stdout:node-exporter.smithi089 smithi089 *:9100 running (97s) 26s ago 10m 9475k - 1.7.0 72c9c2088986 7ef0d1588475 2024-09-21T04:13:42.048 INFO:teuthology.orchestra.run.smithi089.stdout:node-exporter.smithi195 smithi195 *:9100 running (68s) 0s ago 10m 11.4M - 1.7.0 72c9c2088986 61eafd61bc6c 2024-09-21T04:13:42.049 INFO:teuthology.orchestra.run.smithi089.stdout:osd.0 smithi089 running (9m) 26s ago 9m 424M 1935M 17.2.0 e1d6a67b021e db7c0e5020e7 2024-09-21T04:13:42.049 INFO:teuthology.orchestra.run.smithi089.stdout:osd.1 smithi089 running (8m) 26s ago 8m 320M 1935M 17.2.0 e1d6a67b021e f9ec697d08d0 2024-09-21T04:13:42.049 INFO:teuthology.orchestra.run.smithi089.stdout:osd.2 smithi089 running (8m) 26s ago 8m 286M 1935M 17.2.0 e1d6a67b021e cfa000948859 2024-09-21T04:13:42.049 INFO:teuthology.orchestra.run.smithi089.stdout:osd.3 smithi089 running (8m) 26s ago 8m 391M 1935M 17.2.0 e1d6a67b021e 31c610ed5815 2024-09-21T04:13:42.049 INFO:teuthology.orchestra.run.smithi089.stdout:osd.4 smithi195 running (7m) 0s ago 7m 395M 2704M 17.2.0 e1d6a67b021e f00e18a1f220 2024-09-21T04:13:42.049 INFO:teuthology.orchestra.run.smithi089.stdout:osd.5 smithi195 running (7m) 0s ago 7m 442M 2704M 17.2.0 e1d6a67b021e cecfd6ff1eee 2024-09-21T04:13:42.049 INFO:teuthology.orchestra.run.smithi089.stdout:osd.6 smithi195 running (6m) 0s ago 6m 382M 2704M 17.2.0 e1d6a67b021e 0d7db362cc23 2024-09-21T04:13:42.049 INFO:teuthology.orchestra.run.smithi089.stdout:osd.7 smithi195 running (6m) 0s ago 6m 506M 2704M 17.2.0 e1d6a67b021e 0109b3fea188 2024-09-21T04:13:42.049 INFO:teuthology.orchestra.run.smithi089.stdout:prometheus.smithi089 smithi089 *:9095 starting - - - - 2024-09-21T04:13:42.093 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:42 smithi089 ceph-mon[23733]: pgmap v20: 97 pgs: 97 active+clean; 327 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 6.9 KiB/s rd, 4.9 MiB/s wr, 465 op/s 2024-09-21T04:13:42.093 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:42 smithi089 ceph-mon[23733]: from='client.14854 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T04:13:42.093 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:42 smithi089 ceph-mon[23733]: from='mgr.14806 172.21.15.89:0/3152262325' entity='mgr.smithi089.srlyxf' 2024-09-21T04:13:42.093 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:42 smithi089 ceph-mon[23733]: from='mgr.14806 172.21.15.89:0/3152262325' entity='mgr.smithi089.srlyxf' 2024-09-21T04:13:42.328 INFO:teuthology.orchestra.run.smithi089.stdout: 5 118617 10.28 MB/sec execute 141 sec latency 51.196 ms 2024-09-21T04:13:42.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:13:42 smithi195 ceph-mon[25840]: pgmap v20: 97 pgs: 97 active+clean; 327 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 6.9 KiB/s rd, 4.9 MiB/s wr, 465 op/s 2024-09-21T04:13:42.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:13:42 smithi195 ceph-mon[25840]: from='client.14854 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T04:13:42.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:13:42 smithi195 ceph-mon[25840]: from='mgr.14806 172.21.15.89:0/3152262325' entity='mgr.smithi089.srlyxf' 2024-09-21T04:13:42.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:13:42 smithi195 ceph-mon[25840]: from='mgr.14806 172.21.15.89:0/3152262325' entity='mgr.smithi089.srlyxf' 2024-09-21T04:13:42.620 INFO:teuthology.orchestra.run.smithi089.stdout:{ 2024-09-21T04:13:42.620 INFO:teuthology.orchestra.run.smithi089.stdout: "mon": { 2024-09-21T04:13:42.620 INFO:teuthology.orchestra.run.smithi089.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-21T04:13:42.620 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2024-09-21T04:13:42.620 INFO:teuthology.orchestra.run.smithi089.stdout: "mgr": { 2024-09-21T04:13:42.620 INFO:teuthology.orchestra.run.smithi089.stdout: "ceph version 19.3.0-5103-gf36283e7 (f36283e7c91cac1c01cb9a7c4ff21bd697dbd4f2) squid (dev)": 2 2024-09-21T04:13:42.620 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2024-09-21T04:13:42.620 INFO:teuthology.orchestra.run.smithi089.stdout: "osd": { 2024-09-21T04:13:42.620 INFO:teuthology.orchestra.run.smithi089.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-09-21T04:13:42.620 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2024-09-21T04:13:42.620 INFO:teuthology.orchestra.run.smithi089.stdout: "mds": { 2024-09-21T04:13:42.620 INFO:teuthology.orchestra.run.smithi089.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-21T04:13:42.621 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2024-09-21T04:13:42.621 INFO:teuthology.orchestra.run.smithi089.stdout: "overall": { 2024-09-21T04:13:42.621 INFO:teuthology.orchestra.run.smithi089.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 12, 2024-09-21T04:13:42.621 INFO:teuthology.orchestra.run.smithi089.stdout: "ceph version 19.3.0-5103-gf36283e7 (f36283e7c91cac1c01cb9a7c4ff21bd697dbd4f2) squid (dev)": 2 2024-09-21T04:13:42.621 INFO:teuthology.orchestra.run.smithi089.stdout: } 2024-09-21T04:13:42.621 INFO:teuthology.orchestra.run.smithi089.stdout:} 2024-09-21T04:13:43.112 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:43 smithi089 ceph-mon[23733]: from='client.14858 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T04:13:43.112 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:43 smithi089 ceph-mon[23733]: from='client.14862 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T04:13:43.112 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:43 smithi089 ceph-mon[23733]: from='mgr.14806 172.21.15.89:0/3152262325' entity='mgr.smithi089.srlyxf' 2024-09-21T04:13:43.112 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:43 smithi089 ceph-mon[23733]: from='mgr.14806 172.21.15.89:0/3152262325' entity='mgr.smithi089.srlyxf' 2024-09-21T04:13:43.112 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:43 smithi089 ceph-mon[23733]: from='client.? 172.21.15.89:0/1476031969' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T04:13:43.239 INFO:teuthology.orchestra.run.smithi089.stdout:{ 2024-09-21T04:13:43.240 INFO:teuthology.orchestra.run.smithi089.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:2195e5f178be0a9390728e2a14088236d8fe6f82635f5c84418fe37a04457e00", 2024-09-21T04:13:43.240 INFO:teuthology.orchestra.run.smithi089.stdout: "in_progress": true, 2024-09-21T04:13:43.240 INFO:teuthology.orchestra.run.smithi089.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-21T04:13:43.240 INFO:teuthology.orchestra.run.smithi089.stdout: "services_complete": [ 2024-09-21T04:13:43.240 INFO:teuthology.orchestra.run.smithi089.stdout: "mgr" 2024-09-21T04:13:43.240 INFO:teuthology.orchestra.run.smithi089.stdout: ], 2024-09-21T04:13:43.241 INFO:teuthology.orchestra.run.smithi089.stdout: "progress": "2/23 daemons upgraded", 2024-09-21T04:13:43.241 INFO:teuthology.orchestra.run.smithi089.stdout: "message": "Currently upgrading mgr daemons", 2024-09-21T04:13:43.241 INFO:teuthology.orchestra.run.smithi089.stdout: "is_paused": false 2024-09-21T04:13:43.241 INFO:teuthology.orchestra.run.smithi089.stdout:} 2024-09-21T04:13:43.328 INFO:teuthology.orchestra.run.smithi089.stdout: 5 119051 10.27 MB/sec execute 142 sec latency 49.282 ms 2024-09-21T04:13:43.355 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:13:43 smithi195 ceph-mon[25840]: from='client.14858 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T04:13:43.355 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:13:43 smithi195 ceph-mon[25840]: from='client.14862 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T04:13:43.355 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:13:43 smithi195 ceph-mon[25840]: from='mgr.14806 172.21.15.89:0/3152262325' entity='mgr.smithi089.srlyxf' 2024-09-21T04:13:43.355 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:13:43 smithi195 ceph-mon[25840]: from='mgr.14806 172.21.15.89:0/3152262325' entity='mgr.smithi089.srlyxf' 2024-09-21T04:13:43.355 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:13:43 smithi195 ceph-mon[25840]: from='client.? 172.21.15.89:0/1476031969' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T04:13:43.831 INFO:teuthology.orchestra.run.smithi089.stdout:HEALTH_OK 2024-09-21T04:13:44.119 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:44 smithi089 ceph-mon[23733]: pgmap v21: 97 pgs: 97 active+clean; 327 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 6.9 KiB/s rd, 4.9 MiB/s wr, 465 op/s 2024-09-21T04:13:44.119 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:44 smithi089 ceph-mon[23733]: from='mgr.? 172.21.15.195:0/2032481097' entity='mgr.smithi195.cgotzm' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi195.cgotzm/crt"}]: dispatch 2024-09-21T04:13:44.119 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:44 smithi089 ceph-mon[23733]: from='mgr.? 172.21.15.195:0/2032481097' entity='mgr.smithi195.cgotzm' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-21T04:13:44.119 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:44 smithi089 ceph-mon[23733]: from='mgr.? 172.21.15.195:0/2032481097' entity='mgr.smithi195.cgotzm' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi195.cgotzm/key"}]: dispatch 2024-09-21T04:13:44.119 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:44 smithi089 ceph-mon[23733]: from='mgr.? 172.21.15.195:0/2032481097' entity='mgr.smithi195.cgotzm' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-21T04:13:44.120 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:44 smithi089 ceph-mon[23733]: Standby manager daemon smithi195.cgotzm restarted 2024-09-21T04:13:44.120 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:44 smithi089 ceph-mon[23733]: Standby manager daemon smithi195.cgotzm started 2024-09-21T04:13:44.120 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:44 smithi089 ceph-mon[23733]: from='client.? 172.21.15.89:0/3609007219' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-21T04:13:44.328 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:13:44 smithi195 ceph-mon[25840]: pgmap v21: 97 pgs: 97 active+clean; 327 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 6.9 KiB/s rd, 4.9 MiB/s wr, 465 op/s 2024-09-21T04:13:44.328 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:13:44 smithi195 ceph-mon[25840]: from='mgr.? 172.21.15.195:0/2032481097' entity='mgr.smithi195.cgotzm' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi195.cgotzm/crt"}]: dispatch 2024-09-21T04:13:44.328 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:13:44 smithi195 ceph-mon[25840]: from='mgr.? 172.21.15.195:0/2032481097' entity='mgr.smithi195.cgotzm' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-21T04:13:44.328 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:13:44 smithi195 ceph-mon[25840]: from='mgr.? 172.21.15.195:0/2032481097' entity='mgr.smithi195.cgotzm' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi195.cgotzm/key"}]: dispatch 2024-09-21T04:13:44.328 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:13:44 smithi195 ceph-mon[25840]: from='mgr.? 172.21.15.195:0/2032481097' entity='mgr.smithi195.cgotzm' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-21T04:13:44.328 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:13:44 smithi195 ceph-mon[25840]: Standby manager daemon smithi195.cgotzm restarted 2024-09-21T04:13:44.328 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:13:44 smithi195 ceph-mon[25840]: Standby manager daemon smithi195.cgotzm started 2024-09-21T04:13:44.328 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:13:44 smithi195 ceph-mon[25840]: from='client.? 172.21.15.89:0/3609007219' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-21T04:13:44.328 INFO:teuthology.orchestra.run.smithi089.stdout: 5 119532 10.26 MB/sec execute 143 sec latency 48.350 ms 2024-09-21T04:13:45.139 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:45 smithi089 ceph-mon[23733]: from='client.14870 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T04:13:45.140 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:45 smithi089 ceph-mon[23733]: mgrmap e36: smithi089.srlyxf(active, since 37s), standbys: smithi195.cgotzm 2024-09-21T04:13:45.328 INFO:teuthology.orchestra.run.smithi089.stdout: 5 119742 10.23 MB/sec execute 144 sec latency 57.613 ms 2024-09-21T04:13:45.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:13:45 smithi195 ceph-mon[25840]: from='client.14870 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T04:13:45.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:13:45 smithi195 ceph-mon[25840]: mgrmap e36: smithi089.srlyxf(active, since 37s), standbys: smithi195.cgotzm 2024-09-21T04:13:46.180 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:46 smithi089 ceph-mon[23733]: pgmap v22: 97 pgs: 97 active+clean; 317 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 6.3 MiB/s wr, 656 op/s 2024-09-21T04:13:46.180 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:46 smithi089 ceph-mon[23733]: from='mgr.14806 172.21.15.89:0/3152262325' entity='mgr.smithi089.srlyxf' 2024-09-21T04:13:46.180 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:46 smithi089 ceph-mon[23733]: from='mgr.14806 172.21.15.89:0/3152262325' entity='mgr.smithi089.srlyxf' 2024-09-21T04:13:46.328 INFO:teuthology.orchestra.run.smithi089.stdout: 5 120214 10.32 MB/sec execute 145 sec latency 31.245 ms 2024-09-21T04:13:46.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:13:46 smithi195 ceph-mon[25840]: pgmap v22: 97 pgs: 97 active+clean; 317 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 6.3 MiB/s wr, 656 op/s 2024-09-21T04:13:46.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:13:46 smithi195 ceph-mon[25840]: from='mgr.14806 172.21.15.89:0/3152262325' entity='mgr.smithi089.srlyxf' 2024-09-21T04:13:46.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:13:46 smithi195 ceph-mon[25840]: from='mgr.14806 172.21.15.89:0/3152262325' entity='mgr.smithi089.srlyxf' 2024-09-21T04:13:47.328 INFO:teuthology.orchestra.run.smithi089.stdout: 5 120693 10.38 MB/sec execute 146 sec latency 32.491 ms 2024-09-21T04:13:47.476 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:47 smithi089 ceph-mon[23733]: from='mgr.14806 172.21.15.89:0/3152262325' entity='mgr.smithi089.srlyxf' 2024-09-21T04:13:47.477 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:47 smithi089 ceph-mon[23733]: from='mgr.14806 172.21.15.89:0/3152262325' entity='mgr.smithi089.srlyxf' 2024-09-21T04:13:47.477 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:47 smithi089 ceph-mon[23733]: from='mgr.14806 172.21.15.89:0/3152262325' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T04:13:47.477 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:47 smithi089 ceph-mon[23733]: from='mgr.14806 172.21.15.89:0/3152262325' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T04:13:47.477 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:47 smithi089 ceph-mon[23733]: from='mgr.14806 172.21.15.89:0/3152262325' entity='mgr.smithi089.srlyxf' 2024-09-21T04:13:47.477 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:47 smithi089 ceph-mon[23733]: from='mgr.14806 172.21.15.89:0/3152262325' entity='mgr.smithi089.srlyxf' 2024-09-21T04:13:47.477 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:47 smithi089 ceph-mon[23733]: from='mgr.14806 172.21.15.89:0/3152262325' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T04:13:47.477 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:47 smithi089 ceph-mon[23733]: from='mgr.14806 172.21.15.89:0/3152262325' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T04:13:47.477 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:47 smithi089 ceph-mon[23733]: Upgrade: Setting container_image for all mgr 2024-09-21T04:13:47.477 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:47 smithi089 ceph-mon[23733]: from='mgr.14806 172.21.15.89:0/3152262325' entity='mgr.smithi089.srlyxf' 2024-09-21T04:13:47.478 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:47 smithi089 ceph-mon[23733]: from='mgr.14806 172.21.15.89:0/3152262325' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi089.srlyxf"}]: dispatch 2024-09-21T04:13:47.478 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:47 smithi089 ceph-mon[23733]: from='mgr.14806 172.21.15.89:0/3152262325' entity='mgr.smithi089.srlyxf' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi089.srlyxf"}]': finished 2024-09-21T04:13:47.478 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:47 smithi089 ceph-mon[23733]: from='mgr.14806 172.21.15.89:0/3152262325' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi195.cgotzm"}]: dispatch 2024-09-21T04:13:47.478 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:47 smithi089 ceph-mon[23733]: from='mgr.14806 172.21.15.89:0/3152262325' entity='mgr.smithi089.srlyxf' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi195.cgotzm"}]': finished 2024-09-21T04:13:47.478 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:47 smithi089 ceph-mon[23733]: from='mgr.14806 172.21.15.89:0/3152262325' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-09-21T04:13:47.478 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:47 smithi089 ceph-mon[23733]: pgmap v23: 97 pgs: 97 active+clean; 317 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 6.9 KiB/s rd, 4.4 MiB/s wr, 435 op/s 2024-09-21T04:13:47.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:13:47 smithi195 ceph-mon[25840]: from='mgr.14806 172.21.15.89:0/3152262325' entity='mgr.smithi089.srlyxf' 2024-09-21T04:13:47.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:13:47 smithi195 ceph-mon[25840]: from='mgr.14806 172.21.15.89:0/3152262325' entity='mgr.smithi089.srlyxf' 2024-09-21T04:13:47.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:13:47 smithi195 ceph-mon[25840]: from='mgr.14806 172.21.15.89:0/3152262325' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T04:13:47.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:13:47 smithi195 ceph-mon[25840]: from='mgr.14806 172.21.15.89:0/3152262325' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T04:13:47.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:13:47 smithi195 ceph-mon[25840]: from='mgr.14806 172.21.15.89:0/3152262325' entity='mgr.smithi089.srlyxf' 2024-09-21T04:13:47.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:13:47 smithi195 ceph-mon[25840]: from='mgr.14806 172.21.15.89:0/3152262325' entity='mgr.smithi089.srlyxf' 2024-09-21T04:13:47.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:13:47 smithi195 ceph-mon[25840]: from='mgr.14806 172.21.15.89:0/3152262325' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T04:13:47.610 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:13:47 smithi195 ceph-mon[25840]: from='mgr.14806 172.21.15.89:0/3152262325' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T04:13:47.610 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:13:47 smithi195 ceph-mon[25840]: Upgrade: Setting container_image for all mgr 2024-09-21T04:13:47.610 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:13:47 smithi195 ceph-mon[25840]: from='mgr.14806 172.21.15.89:0/3152262325' entity='mgr.smithi089.srlyxf' 2024-09-21T04:13:47.610 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:13:47 smithi195 ceph-mon[25840]: from='mgr.14806 172.21.15.89:0/3152262325' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi089.srlyxf"}]: dispatch 2024-09-21T04:13:47.610 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:13:47 smithi195 ceph-mon[25840]: from='mgr.14806 172.21.15.89:0/3152262325' entity='mgr.smithi089.srlyxf' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi089.srlyxf"}]': finished 2024-09-21T04:13:47.610 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:13:47 smithi195 ceph-mon[25840]: from='mgr.14806 172.21.15.89:0/3152262325' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi195.cgotzm"}]: dispatch 2024-09-21T04:13:47.610 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:13:47 smithi195 ceph-mon[25840]: from='mgr.14806 172.21.15.89:0/3152262325' entity='mgr.smithi089.srlyxf' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi195.cgotzm"}]': finished 2024-09-21T04:13:47.610 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:13:47 smithi195 ceph-mon[25840]: from='mgr.14806 172.21.15.89:0/3152262325' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-09-21T04:13:47.610 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:13:47 smithi195 ceph-mon[25840]: pgmap v23: 97 pgs: 97 active+clean; 317 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 6.9 KiB/s rd, 4.4 MiB/s wr, 435 op/s 2024-09-21T04:13:48.328 INFO:teuthology.orchestra.run.smithi089.stdout: 5 120991 10.37 MB/sec execute 147 sec latency 36.497 ms 2024-09-21T04:13:49.336 INFO:teuthology.orchestra.run.smithi089.stdout: 5 121457 10.33 MB/sec execute 148 sec latency 29.324 ms 2024-09-21T04:13:50.019 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:49 smithi089 ceph-mon[23733]: Upgrade: Updating mon.smithi089 2024-09-21T04:13:50.020 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:49 smithi089 ceph-mon[23733]: from='mgr.14806 172.21.15.89:0/3152262325' entity='mgr.smithi089.srlyxf' 2024-09-21T04:13:50.020 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:49 smithi089 ceph-mon[23733]: from='mgr.14806 172.21.15.89:0/3152262325' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-21T04:13:50.020 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:49 smithi089 ceph-mon[23733]: from='mgr.14806 172.21.15.89:0/3152262325' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-21T04:13:50.020 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:49 smithi089 ceph-mon[23733]: from='mgr.14806 172.21.15.89:0/3152262325' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T04:13:50.020 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:49 smithi089 ceph-mon[23733]: Deploying daemon mon.smithi089 on smithi089 2024-09-21T04:13:50.020 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:49 smithi089 ceph-mon[23733]: pgmap v24: 97 pgs: 97 active+clean; 327 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 9.1 KiB/s rd, 7.2 MiB/s wr, 658 op/s 2024-09-21T04:13:50.336 INFO:teuthology.orchestra.run.smithi089.stdout: 5 122155 10.39 MB/sec execute 149 sec latency 20.754 ms 2024-09-21T04:13:50.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:13:49 smithi195 ceph-mon[25840]: Upgrade: Updating mon.smithi089 2024-09-21T04:13:50.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:13:49 smithi195 ceph-mon[25840]: from='mgr.14806 172.21.15.89:0/3152262325' entity='mgr.smithi089.srlyxf' 2024-09-21T04:13:50.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:13:49 smithi195 ceph-mon[25840]: from='mgr.14806 172.21.15.89:0/3152262325' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-21T04:13:50.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:13:49 smithi195 ceph-mon[25840]: from='mgr.14806 172.21.15.89:0/3152262325' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-21T04:13:50.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:13:49 smithi195 ceph-mon[25840]: from='mgr.14806 172.21.15.89:0/3152262325' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T04:13:50.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:13:49 smithi195 ceph-mon[25840]: Deploying daemon mon.smithi089 on smithi089 2024-09-21T04:13:50.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:13:49 smithi195 ceph-mon[25840]: pgmap v24: 97 pgs: 97 active+clean; 327 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 9.1 KiB/s rd, 7.2 MiB/s wr, 658 op/s 2024-09-21T04:13:51.336 INFO:teuthology.orchestra.run.smithi089.stdout: 5 122672 10.36 MB/sec execute 150 sec latency 41.192 ms 2024-09-21T04:13:51.707 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:51 smithi089 systemd[1]: Stopping Ceph mon.smithi089 for 1b9b79fc-77ce-11ef-baf5-efdc52797490... 2024-09-21T04:13:52.213 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:51 smithi089 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-mon-smithi089[23729]: 2024-09-21T04:13:51.985+0000 7fa3b5148700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.smithi089 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-09-21T04:13:52.213 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:51 smithi089 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-mon-smithi089[23729]: 2024-09-21T04:13:51.985+0000 7fa3b5148700 -1 mon.smithi089@0(leader) e2 *** Got Signal Terminated *** 2024-09-21T04:13:52.213 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:52 smithi089 podman[63910]: 2024-09-21 04:13:52.017046478 +0000 UTC m=+0.159283388 container died 1d2b3719e46efeb28714f3f5bdfa041021ff4c3c0f21316359a357c8b7c7b7d9 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-mon-smithi089, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_CLEAN=True, GIT_REPO=https://github.com/ceph/ceph-container.git, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, vcs-type=git, RELEASE=HEAD, vendor=Red Hat, Inc., maintainer=Guillaume Abrioux , architecture=x86_64, release=754, GIT_BRANCH=HEAD, io.openshift.tags=base centos centos-stream, distribution-scope=public, version=8, io.buildah.version=1.19.8, CEPH_POINT_RELEASE=-17.2.0, build-date=2022-05-03T08:36:31.336870, io.openshift.expose-services=, name=centos-stream, ceph=True, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.k8s.display-name=CentOS Stream 8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.component=centos-stream-container) 2024-09-21T04:13:52.336 INFO:teuthology.orchestra.run.smithi089.stdout: 5 123120 10.36 MB/sec execute 151 sec latency 50.437 ms 2024-09-21T04:13:52.969 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:52 smithi089 podman[63910]: 2024-09-21 04:13:52.889241111 +0000 UTC m=+1.031478015 container cleanup 1d2b3719e46efeb28714f3f5bdfa041021ff4c3c0f21316359a357c8b7c7b7d9 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-mon-smithi089, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, release=754, distribution-scope=public, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.openshift.expose-services=, vendor=Red Hat, Inc., io.buildah.version=1.19.8, architecture=x86_64, name=centos-stream, maintainer=Guillaume Abrioux , RELEASE=HEAD, GIT_CLEAN=True, io.openshift.tags=base centos centos-stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, version=8, CEPH_POINT_RELEASE=-17.2.0, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_REPO=https://github.com/ceph/ceph-container.git, vcs-type=git, io.k8s.display-name=CentOS Stream 8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., build-date=2022-05-03T08:36:31.336870, com.redhat.component=centos-stream-container, ceph=True, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_BRANCH=HEAD) 2024-09-21T04:13:52.969 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:52 smithi089 bash[63910]: ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-mon-smithi089 2024-09-21T04:13:53.337 INFO:teuthology.orchestra.run.smithi089.stdout: 5 123641 10.47 MB/sec execute 152 sec latency 30.562 ms 2024-09-21T04:13:53.471 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:53 smithi089 podman[63921]: 2024-09-21 04:13:53.259678029 +0000 UTC m=+1.236149737 container remove 1d2b3719e46efeb28714f3f5bdfa041021ff4c3c0f21316359a357c8b7c7b7d9 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-mon-smithi089, distribution-scope=public, io.k8s.display-name=CentOS Stream 8, version=8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vcs-type=git, ceph=True, CEPH_POINT_RELEASE=-17.2.0, maintainer=Guillaume Abrioux , io.buildah.version=1.19.8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., RELEASE=HEAD, architecture=x86_64, release=754, build-date=2022-05-03T08:36:31.336870, GIT_BRANCH=HEAD, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vendor=Red Hat, Inc., io.openshift.expose-services=, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, name=centos-stream, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.component=centos-stream-container, GIT_CLEAN=True, io.openshift.tags=base centos centos-stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image) 2024-09-21T04:13:53.722 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:53 smithi089 systemd[1]: ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490@mon.smithi089.service: Deactivated successfully. 2024-09-21T04:13:53.722 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:53 smithi089 systemd[1]: Stopped Ceph mon.smithi089 for 1b9b79fc-77ce-11ef-baf5-efdc52797490. 2024-09-21T04:13:53.722 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:53 smithi089 systemd[1]: ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490@mon.smithi089.service: Consumed 15.576s CPU time. 2024-09-21T04:13:54.224 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:54 smithi089 systemd[1]: Starting Ceph mon.smithi089 for 1b9b79fc-77ce-11ef-baf5-efdc52797490... 2024-09-21T04:13:54.337 INFO:teuthology.orchestra.run.smithi089.stdout: 5 124424 10.58 MB/sec execute 153 sec latency 22.237 ms 2024-09-21T04:13:54.475 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:54 smithi089 podman[64057]: 2024-09-21 04:13:54.209314353 +0000 UTC m=+0.023948937 image pull 48ed0a2cf89d6d0371a2c6185a5869a747265f6abd5ac2a74f00cc32d5628721 quay.ceph.io/ceph-ci/ceph@sha256:2195e5f178be0a9390728e2a14088236d8fe6f82635f5c84418fe37a04457e00 2024-09-21T04:13:54.475 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:54 smithi089 podman[64057]: 2024-09-21 04:13:54.377260585 +0000 UTC m=+0.191895155 container create 7dd34c8a6f21e84b210383e6f843ee23f5d95364b9b5c161c29c2892088e3af1 (image=quay.ceph.io/ceph-ci/ceph@sha256:2195e5f178be0a9390728e2a14088236d8fe6f82635f5c84418fe37a04457e00, name=ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-mon-smithi089, maintainer=Guillaume Abrioux , RELEASE=wip-guits-main-2024-09-20-2310-f36283e, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23) 2024-09-21T04:13:54.997 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:54 smithi089 podman[64057]: 2024-09-21 04:13:54.773314105 +0000 UTC m=+0.587948670 container init 7dd34c8a6f21e84b210383e6f843ee23f5d95364b9b5c161c29c2892088e3af1 (image=quay.ceph.io/ceph-ci/ceph@sha256:2195e5f178be0a9390728e2a14088236d8fe6f82635f5c84418fe37a04457e00, name=ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-mon-smithi089, io.buildah.version=1.37.2, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, RELEASE=wip-guits-main-2024-09-20-2310-f36283e, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, GIT_CLEAN=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2) 2024-09-21T04:13:54.997 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:54 smithi089 podman[64057]: 2024-09-21 04:13:54.786095922 +0000 UTC m=+0.600730487 container start 7dd34c8a6f21e84b210383e6f843ee23f5d95364b9b5c161c29c2892088e3af1 (image=quay.ceph.io/ceph-ci/ceph@sha256:2195e5f178be0a9390728e2a14088236d8fe6f82635f5c84418fe37a04457e00, name=ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-mon-smithi089, maintainer=Guillaume Abrioux , RELEASE=wip-guits-main-2024-09-20-2310-f36283e, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, org.label-schema.license=GPLv2) 2024-09-21T04:13:54.997 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:54 smithi089 ceph-mon[64076]: set uid:gid to 167:167 (ceph:ceph) 2024-09-21T04:13:54.998 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:54 smithi089 ceph-mon[64076]: ceph version 19.3.0-5103-gf36283e7 (f36283e7c91cac1c01cb9a7c4ff21bd697dbd4f2) squid (dev), process ceph-mon, pid 2 2024-09-21T04:13:54.998 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:54 smithi089 ceph-mon[64076]: pidfile_write: ignore empty --pid-file 2024-09-21T04:13:54.998 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:54 smithi089 ceph-mon[64076]: load: jerasure load: lrc 2024-09-21T04:13:54.998 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:54 smithi089 ceph-mon[64076]: rocksdb: RocksDB version: 7.9.2 2024-09-21T04:13:54.998 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:54 smithi089 ceph-mon[64076]: rocksdb: Git sha 0 2024-09-21T04:13:54.998 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:54 smithi089 ceph-mon[64076]: rocksdb: Compile date 2024-09-20 23:32:13 2024-09-21T04:13:54.998 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:54 smithi089 ceph-mon[64076]: rocksdb: DB SUMMARY 2024-09-21T04:13:54.998 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:54 smithi089 ceph-mon[64076]: rocksdb: DB Session ID: VULAKVJBO0F1TJ74V1FE 2024-09-21T04:13:54.998 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:54 smithi089 ceph-mon[64076]: rocksdb: CURRENT file: CURRENT 2024-09-21T04:13:54.998 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:54 smithi089 ceph-mon[64076]: rocksdb: IDENTITY file: IDENTITY 2024-09-21T04:13:54.998 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:54 smithi089 ceph-mon[64076]: rocksdb: MANIFEST file: MANIFEST-000015 size: 979 Bytes 2024-09-21T04:13:54.998 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:54 smithi089 ceph-mon[64076]: rocksdb: SST files in /var/lib/ceph/mon/ceph-smithi089/store.db dir, Total Num: 1, files: 000030.sst 2024-09-21T04:13:54.999 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:54 smithi089 ceph-mon[64076]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-smithi089/store.db: 000028.log size: 1640641 ; 2024-09-21T04:13:54.999 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:54 smithi089 ceph-mon[64076]: rocksdb: Options.error_if_exists: 0 2024-09-21T04:13:54.999 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:54 smithi089 ceph-mon[64076]: rocksdb: Options.create_if_missing: 0 2024-09-21T04:13:54.999 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:54 smithi089 ceph-mon[64076]: rocksdb: Options.paranoid_checks: 1 2024-09-21T04:13:54.999 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:54 smithi089 ceph-mon[64076]: rocksdb: Options.flush_verify_memtable_count: 1 2024-09-21T04:13:54.999 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:54 smithi089 ceph-mon[64076]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-09-21T04:13:54.999 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:54 smithi089 ceph-mon[64076]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-09-21T04:13:54.999 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:54 smithi089 ceph-mon[64076]: rocksdb: Options.env: 0x55dcec744120 2024-09-21T04:13:54.999 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:54 smithi089 ceph-mon[64076]: rocksdb: Options.fs: PosixFileSystem 2024-09-21T04:13:54.999 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:54 smithi089 ceph-mon[64076]: rocksdb: Options.info_log: 0x55dcee2558e0 2024-09-21T04:13:54.999 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:54 smithi089 ceph-mon[64076]: rocksdb: Options.max_file_opening_threads: 16 2024-09-21T04:13:55.001 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:54 smithi089 ceph-mon[64076]: rocksdb: Options.statistics: (nil) 2024-09-21T04:13:55.001 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:54 smithi089 ceph-mon[64076]: rocksdb: Options.use_fsync: 0 2024-09-21T04:13:55.001 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:54 smithi089 ceph-mon[64076]: rocksdb: Options.max_log_file_size: 0 2024-09-21T04:13:55.001 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:54 smithi089 ceph-mon[64076]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-09-21T04:13:55.001 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:54 smithi089 ceph-mon[64076]: rocksdb: Options.log_file_time_to_roll: 0 2024-09-21T04:13:55.001 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:54 smithi089 ceph-mon[64076]: rocksdb: Options.keep_log_file_num: 1000 2024-09-21T04:13:55.001 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:54 smithi089 ceph-mon[64076]: rocksdb: Options.recycle_log_file_num: 0 2024-09-21T04:13:55.001 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:54 smithi089 ceph-mon[64076]: rocksdb: Options.allow_fallocate: 1 2024-09-21T04:13:55.001 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:54 smithi089 ceph-mon[64076]: rocksdb: Options.allow_mmap_reads: 0 2024-09-21T04:13:55.001 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:54 smithi089 ceph-mon[64076]: rocksdb: Options.allow_mmap_writes: 0 2024-09-21T04:13:55.001 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:54 smithi089 ceph-mon[64076]: rocksdb: Options.use_direct_reads: 0 2024-09-21T04:13:55.002 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:54 smithi089 ceph-mon[64076]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-09-21T04:13:55.002 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:54 smithi089 ceph-mon[64076]: rocksdb: Options.create_missing_column_families: 0 2024-09-21T04:13:55.002 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:54 smithi089 ceph-mon[64076]: rocksdb: Options.db_log_dir: 2024-09-21T04:13:55.002 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:54 smithi089 ceph-mon[64076]: rocksdb: Options.wal_dir: 2024-09-21T04:13:55.002 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:54 smithi089 ceph-mon[64076]: rocksdb: Options.table_cache_numshardbits: 6 2024-09-21T04:13:55.002 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:54 smithi089 ceph-mon[64076]: rocksdb: Options.WAL_ttl_seconds: 0 2024-09-21T04:13:55.002 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:54 smithi089 ceph-mon[64076]: rocksdb: Options.WAL_size_limit_MB: 0 2024-09-21T04:13:55.002 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:54 smithi089 ceph-mon[64076]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-09-21T04:13:55.002 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:54 smithi089 ceph-mon[64076]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-09-21T04:13:55.002 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:54 smithi089 ceph-mon[64076]: rocksdb: Options.is_fd_close_on_exec: 1 2024-09-21T04:13:55.002 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:54 smithi089 ceph-mon[64076]: rocksdb: Options.advise_random_on_open: 1 2024-09-21T04:13:55.002 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:54 smithi089 ceph-mon[64076]: rocksdb: Options.db_write_buffer_size: 0 2024-09-21T04:13:55.003 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:54 smithi089 ceph-mon[64076]: rocksdb: Options.write_buffer_manager: 0x55dcee259b80 2024-09-21T04:13:55.003 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:54 smithi089 ceph-mon[64076]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-09-21T04:13:55.003 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:54 smithi089 ceph-mon[64076]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-09-21T04:13:55.003 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:54 smithi089 ceph-mon[64076]: rocksdb: Options.use_adaptive_mutex: 0 2024-09-21T04:13:55.003 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:54 smithi089 ceph-mon[64076]: rocksdb: Options.rate_limiter: (nil) 2024-09-21T04:13:55.003 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:54 smithi089 ceph-mon[64076]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-09-21T04:13:55.003 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:54 smithi089 ceph-mon[64076]: rocksdb: Options.wal_recovery_mode: 2 2024-09-21T04:13:55.003 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:54 smithi089 ceph-mon[64076]: rocksdb: Options.enable_thread_tracking: 0 2024-09-21T04:13:55.003 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:54 smithi089 ceph-mon[64076]: rocksdb: Options.enable_pipelined_write: 0 2024-09-21T04:13:55.004 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:54 smithi089 ceph-mon[64076]: rocksdb: Options.unordered_write: 0 2024-09-21T04:13:55.004 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:54 smithi089 ceph-mon[64076]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-09-21T04:13:55.004 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:54 smithi089 ceph-mon[64076]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-09-21T04:13:55.004 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:54 smithi089 ceph-mon[64076]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-09-21T04:13:55.004 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:54 smithi089 ceph-mon[64076]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-09-21T04:13:55.004 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:54 smithi089 ceph-mon[64076]: rocksdb: Options.row_cache: None 2024-09-21T04:13:55.004 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:54 smithi089 ceph-mon[64076]: rocksdb: Options.wal_filter: None 2024-09-21T04:13:55.004 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:54 smithi089 ceph-mon[64076]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-09-21T04:13:55.004 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:54 smithi089 ceph-mon[64076]: rocksdb: Options.allow_ingest_behind: 0 2024-09-21T04:13:55.004 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:54 smithi089 ceph-mon[64076]: rocksdb: Options.two_write_queues: 0 2024-09-21T04:13:55.004 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:54 smithi089 ceph-mon[64076]: rocksdb: Options.manual_wal_flush: 0 2024-09-21T04:13:55.004 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:54 smithi089 ceph-mon[64076]: rocksdb: Options.wal_compression: 0 2024-09-21T04:13:55.005 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:54 smithi089 ceph-mon[64076]: rocksdb: Options.atomic_flush: 0 2024-09-21T04:13:55.005 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:54 smithi089 ceph-mon[64076]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-09-21T04:13:55.005 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:54 smithi089 ceph-mon[64076]: rocksdb: Options.persist_stats_to_disk: 0 2024-09-21T04:13:55.005 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:54 smithi089 ceph-mon[64076]: rocksdb: Options.write_dbid_to_manifest: 0 2024-09-21T04:13:55.005 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:54 smithi089 ceph-mon[64076]: rocksdb: Options.log_readahead_size: 0 2024-09-21T04:13:55.005 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:54 smithi089 ceph-mon[64076]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-09-21T04:13:55.005 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:54 smithi089 ceph-mon[64076]: rocksdb: Options.best_efforts_recovery: 0 2024-09-21T04:13:55.005 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:54 smithi089 ceph-mon[64076]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-09-21T04:13:55.005 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:54 smithi089 ceph-mon[64076]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-09-21T04:13:55.005 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:54 smithi089 ceph-mon[64076]: rocksdb: Options.allow_data_in_errors: 0 2024-09-21T04:13:55.005 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:54 smithi089 ceph-mon[64076]: rocksdb: Options.db_host_id: __hostname__ 2024-09-21T04:13:55.005 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:54 smithi089 ceph-mon[64076]: rocksdb: Options.enforce_single_del_contracts: true 2024-09-21T04:13:55.005 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:54 smithi089 ceph-mon[64076]: rocksdb: Options.max_background_jobs: 2 2024-09-21T04:13:55.006 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:54 smithi089 ceph-mon[64076]: rocksdb: Options.max_background_compactions: -1 2024-09-21T04:13:55.006 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:54 smithi089 ceph-mon[64076]: rocksdb: Options.max_subcompactions: 1 2024-09-21T04:13:55.006 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:54 smithi089 ceph-mon[64076]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-09-21T04:13:55.006 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:54 smithi089 ceph-mon[64076]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-09-21T04:13:55.006 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:54 smithi089 ceph-mon[64076]: rocksdb: Options.delayed_write_rate : 16777216 2024-09-21T04:13:55.006 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:54 smithi089 ceph-mon[64076]: rocksdb: Options.max_total_wal_size: 0 2024-09-21T04:13:55.006 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:54 smithi089 ceph-mon[64076]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-09-21T04:13:55.006 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:54 smithi089 ceph-mon[64076]: rocksdb: Options.stats_dump_period_sec: 600 2024-09-21T04:13:55.006 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:54 smithi089 ceph-mon[64076]: rocksdb: Options.stats_persist_period_sec: 600 2024-09-21T04:13:55.006 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:54 smithi089 ceph-mon[64076]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-09-21T04:13:55.006 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:54 smithi089 ceph-mon[64076]: rocksdb: Options.max_open_files: -1 2024-09-21T04:13:55.006 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:54 smithi089 ceph-mon[64076]: rocksdb: Options.bytes_per_sync: 0 2024-09-21T04:13:55.006 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:54 smithi089 ceph-mon[64076]: rocksdb: Options.wal_bytes_per_sync: 0 2024-09-21T04:13:55.007 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:54 smithi089 ceph-mon[64076]: rocksdb: Options.strict_bytes_per_sync: 0 2024-09-21T04:13:55.007 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:54 smithi089 ceph-mon[64076]: rocksdb: Options.compaction_readahead_size: 0 2024-09-21T04:13:55.007 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:54 smithi089 ceph-mon[64076]: rocksdb: Options.max_background_flushes: -1 2024-09-21T04:13:55.007 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:54 smithi089 ceph-mon[64076]: rocksdb: Compression algorithms supported: 2024-09-21T04:13:55.007 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:54 smithi089 ceph-mon[64076]: rocksdb: kZSTD supported: 0 2024-09-21T04:13:55.007 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:54 smithi089 ceph-mon[64076]: rocksdb: kXpressCompression supported: 0 2024-09-21T04:13:55.007 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:54 smithi089 ceph-mon[64076]: rocksdb: kBZip2Compression supported: 0 2024-09-21T04:13:55.007 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:54 smithi089 ceph-mon[64076]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-09-21T04:13:55.007 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:54 smithi089 ceph-mon[64076]: rocksdb: kLZ4Compression supported: 1 2024-09-21T04:13:55.007 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:54 smithi089 ceph-mon[64076]: rocksdb: kZlibCompression supported: 1 2024-09-21T04:13:55.007 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:54 smithi089 ceph-mon[64076]: rocksdb: kLZ4HCCompression supported: 1 2024-09-21T04:13:55.007 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:54 smithi089 ceph-mon[64076]: rocksdb: kSnappyCompression supported: 1 2024-09-21T04:13:55.008 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:54 smithi089 ceph-mon[64076]: rocksdb: Fast CRC32 supported: Supported on x86 2024-09-21T04:13:55.008 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:54 smithi089 ceph-mon[64076]: rocksdb: DMutex implementation: pthread_mutex_t 2024-09-21T04:13:55.008 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:54 smithi089 ceph-mon[64076]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-smithi089/store.db/MANIFEST-000015 2024-09-21T04:13:55.008 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:54 smithi089 ceph-mon[64076]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-09-21T04:13:55.008 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:54 smithi089 ceph-mon[64076]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-09-21T04:13:55.008 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:54 smithi089 ceph-mon[64076]: rocksdb: Options.merge_operator: 2024-09-21T04:13:55.008 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:54 smithi089 ceph-mon[64076]: rocksdb: Options.compaction_filter: None 2024-09-21T04:13:55.008 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:54 smithi089 ceph-mon[64076]: rocksdb: Options.compaction_filter_factory: None 2024-09-21T04:13:55.008 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:54 smithi089 ceph-mon[64076]: rocksdb: Options.sst_partitioner_factory: None 2024-09-21T04:13:55.008 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:54 smithi089 ceph-mon[64076]: rocksdb: Options.memtable_factory: SkipListFactory 2024-09-21T04:13:55.008 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:54 smithi089 ceph-mon[64076]: rocksdb: Options.table_factory: BlockBasedTable 2024-09-21T04:13:55.008 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:54 smithi089 ceph-mon[64076]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55dcee255540) 2024-09-21T04:13:55.009 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout: cache_index_and_filter_blocks: 1 2024-09-21T04:13:55.009 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-09-21T04:13:55.009 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-09-21T04:13:55.009 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout: pin_top_level_index_and_filter: 1 2024-09-21T04:13:55.009 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout: index_type: 0 2024-09-21T04:13:55.009 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout: data_block_index_type: 0 2024-09-21T04:13:55.009 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout: index_shortening: 1 2024-09-21T04:13:55.009 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout: data_block_hash_table_util_ratio: 0.750000 2024-09-21T04:13:55.009 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout: checksum: 4 2024-09-21T04:13:55.009 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout: no_block_cache: 0 2024-09-21T04:13:55.009 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout: block_cache: 0x55dcee249610 2024-09-21T04:13:55.009 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout: block_cache_name: BinnedLRUCache 2024-09-21T04:13:55.010 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout: block_cache_options: 2024-09-21T04:13:55.010 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout: capacity : 536870912 2024-09-21T04:13:55.010 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout: num_shard_bits : 4 2024-09-21T04:13:55.010 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout: strict_capacity_limit : 0 2024-09-21T04:13:55.010 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout: high_pri_pool_ratio: 0.000 2024-09-21T04:13:55.010 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout: block_cache_compressed: (nil) 2024-09-21T04:13:55.010 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout: persistent_cache: (nil) 2024-09-21T04:13:55.010 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout: block_size: 4096 2024-09-21T04:13:55.010 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout: block_size_deviation: 10 2024-09-21T04:13:55.010 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout: block_restart_interval: 16 2024-09-21T04:13:55.011 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout: index_block_restart_interval: 1 2024-09-21T04:13:55.011 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout: metadata_block_size: 4096 2024-09-21T04:13:55.011 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout: partition_filters: 0 2024-09-21T04:13:55.011 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout: use_delta_encoding: 1 2024-09-21T04:13:55.011 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout: filter_policy: bloomfilter 2024-09-21T04:13:55.011 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout: whole_key_filtering: 1 2024-09-21T04:13:55.011 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout: verify_compression: 0 2024-09-21T04:13:55.011 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout: read_amp_bytes_per_bit: 0 2024-09-21T04:13:55.011 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout: format_version: 5 2024-09-21T04:13:55.011 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout: enable_index_compression: 1 2024-09-21T04:13:55.011 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout: block_align: 0 2024-09-21T04:13:55.012 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout: max_auto_readahead_size: 262144 2024-09-21T04:13:55.012 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout: prepopulate_block_cache: 0 2024-09-21T04:13:55.012 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout: initial_auto_readahead_size: 8192 2024-09-21T04:13:55.012 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout: num_file_reads_for_auto_readahead: 2 2024-09-21T04:13:55.012 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:54 smithi089 ceph-mon[64076]: rocksdb: Options.write_buffer_size: 33554432 2024-09-21T04:13:55.012 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:54 smithi089 ceph-mon[64076]: rocksdb: Options.max_write_buffer_number: 2 2024-09-21T04:13:55.012 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:54 smithi089 ceph-mon[64076]: rocksdb: Options.compression: NoCompression 2024-09-21T04:13:55.012 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:54 smithi089 ceph-mon[64076]: rocksdb: Options.bottommost_compression: Disabled 2024-09-21T04:13:55.012 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:54 smithi089 ceph-mon[64076]: rocksdb: Options.prefix_extractor: nullptr 2024-09-21T04:13:55.012 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:54 smithi089 ceph-mon[64076]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-09-21T04:13:55.012 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:54 smithi089 ceph-mon[64076]: rocksdb: Options.num_levels: 7 2024-09-21T04:13:55.012 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:54 smithi089 ceph-mon[64076]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-09-21T04:13:55.012 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:54 smithi089 ceph-mon[64076]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-09-21T04:13:55.013 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:54 smithi089 ceph-mon[64076]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-09-21T04:13:55.013 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:54 smithi089 ceph-mon[64076]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-09-21T04:13:55.013 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:54 smithi089 ceph-mon[64076]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-09-21T04:13:55.013 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:54 smithi089 ceph-mon[64076]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-09-21T04:13:55.013 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:54 smithi089 ceph-mon[64076]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-09-21T04:13:55.013 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:54 smithi089 ceph-mon[64076]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-09-21T04:13:55.013 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:54 smithi089 ceph-mon[64076]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-09-21T04:13:55.013 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:54 smithi089 ceph-mon[64076]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-09-21T04:13:55.013 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:54 smithi089 ceph-mon[64076]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-09-21T04:13:55.013 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:54 smithi089 ceph-mon[64076]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-09-21T04:13:55.013 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:54 smithi089 ceph-mon[64076]: rocksdb: Options.compression_opts.window_bits: -14 2024-09-21T04:13:55.013 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:54 smithi089 ceph-mon[64076]: rocksdb: Options.compression_opts.level: 32767 2024-09-21T04:13:55.013 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:54 smithi089 ceph-mon[64076]: rocksdb: Options.compression_opts.strategy: 0 2024-09-21T04:13:55.014 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:54 smithi089 ceph-mon[64076]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-09-21T04:13:55.014 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:54 smithi089 ceph-mon[64076]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-09-21T04:13:55.014 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:54 smithi089 ceph-mon[64076]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-09-21T04:13:55.014 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:54 smithi089 ceph-mon[64076]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-09-21T04:13:55.014 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:54 smithi089 ceph-mon[64076]: rocksdb: Options.compression_opts.enabled: false 2024-09-21T04:13:55.014 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:54 smithi089 ceph-mon[64076]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-09-21T04:13:55.014 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:54 smithi089 ceph-mon[64076]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-09-21T04:13:55.014 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:54 smithi089 ceph-mon[64076]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-09-21T04:13:55.014 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:54 smithi089 ceph-mon[64076]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-09-21T04:13:55.014 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:54 smithi089 ceph-mon[64076]: rocksdb: Options.target_file_size_base: 67108864 2024-09-21T04:13:55.014 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:54 smithi089 ceph-mon[64076]: rocksdb: Options.target_file_size_multiplier: 1 2024-09-21T04:13:55.014 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:54 smithi089 ceph-mon[64076]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-09-21T04:13:55.015 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:54 smithi089 ceph-mon[64076]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-09-21T04:13:55.015 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:54 smithi089 ceph-mon[64076]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-09-21T04:13:55.015 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:54 smithi089 ceph-mon[64076]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-09-21T04:13:55.015 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:54 smithi089 ceph-mon[64076]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-09-21T04:13:55.015 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:54 smithi089 ceph-mon[64076]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-09-21T04:13:55.015 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:54 smithi089 ceph-mon[64076]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-09-21T04:13:55.015 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:54 smithi089 ceph-mon[64076]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-09-21T04:13:55.015 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:54 smithi089 ceph-mon[64076]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-09-21T04:13:55.015 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:54 smithi089 ceph-mon[64076]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-09-21T04:13:55.015 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:54 smithi089 ceph-mon[64076]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-09-21T04:13:55.015 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:54 smithi089 ceph-mon[64076]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-09-21T04:13:55.015 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:54 smithi089 ceph-mon[64076]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-09-21T04:13:55.015 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:54 smithi089 ceph-mon[64076]: rocksdb: Options.arena_block_size: 1048576 2024-09-21T04:13:55.016 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:54 smithi089 ceph-mon[64076]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-09-21T04:13:55.016 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:54 smithi089 ceph-mon[64076]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-09-21T04:13:55.016 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:54 smithi089 ceph-mon[64076]: rocksdb: Options.disable_auto_compactions: 0 2024-09-21T04:13:55.016 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:54 smithi089 ceph-mon[64076]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-09-21T04:13:55.016 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:54 smithi089 ceph-mon[64076]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-09-21T04:13:55.016 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:54 smithi089 ceph-mon[64076]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-09-21T04:13:55.016 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:54 smithi089 ceph-mon[64076]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-09-21T04:13:55.016 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:54 smithi089 ceph-mon[64076]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-09-21T04:13:55.016 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:54 smithi089 ceph-mon[64076]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-09-21T04:13:55.016 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:54 smithi089 ceph-mon[64076]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-09-21T04:13:55.016 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:54 smithi089 ceph-mon[64076]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-09-21T04:13:55.016 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:54 smithi089 ceph-mon[64076]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-09-21T04:13:55.016 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:54 smithi089 ceph-mon[64076]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-09-21T04:13:55.017 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:54 smithi089 ceph-mon[64076]: rocksdb: Options.table_properties_collectors: CompactOnDeletionCollector (Sliding window size = 32768 Deletion trigger = 16384 Deletion ratio = 0); 2024-09-21T04:13:55.017 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:54 smithi089 ceph-mon[64076]: rocksdb: Options.inplace_update_support: 0 2024-09-21T04:13:55.017 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:54 smithi089 ceph-mon[64076]: rocksdb: Options.inplace_update_num_locks: 10000 2024-09-21T04:13:55.017 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:54 smithi089 ceph-mon[64076]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-09-21T04:13:55.017 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:54 smithi089 ceph-mon[64076]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-09-21T04:13:55.017 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:54 smithi089 ceph-mon[64076]: rocksdb: Options.memtable_huge_page_size: 0 2024-09-21T04:13:55.017 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:54 smithi089 ceph-mon[64076]: rocksdb: Options.bloom_locality: 0 2024-09-21T04:13:55.017 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:54 smithi089 ceph-mon[64076]: rocksdb: Options.max_successive_merges: 0 2024-09-21T04:13:55.017 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:54 smithi089 ceph-mon[64076]: rocksdb: Options.optimize_filters_for_hits: 0 2024-09-21T04:13:55.018 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:54 smithi089 ceph-mon[64076]: rocksdb: Options.paranoid_file_checks: 0 2024-09-21T04:13:55.018 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:54 smithi089 ceph-mon[64076]: rocksdb: Options.force_consistency_checks: 1 2024-09-21T04:13:55.018 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:54 smithi089 ceph-mon[64076]: rocksdb: Options.report_bg_io_stats: 0 2024-09-21T04:13:55.018 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:54 smithi089 ceph-mon[64076]: rocksdb: Options.ttl: 2592000 2024-09-21T04:13:55.018 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:54 smithi089 ceph-mon[64076]: rocksdb: Options.periodic_compaction_seconds: 0 2024-09-21T04:13:55.018 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:54 smithi089 ceph-mon[64076]: rocksdb: Options.preclude_last_level_data_seconds: 0 2024-09-21T04:13:55.018 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:54 smithi089 ceph-mon[64076]: rocksdb: Options.preserve_internal_time_seconds: 0 2024-09-21T04:13:55.018 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:54 smithi089 ceph-mon[64076]: rocksdb: Options.enable_blob_files: false 2024-09-21T04:13:55.018 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:54 smithi089 ceph-mon[64076]: rocksdb: Options.min_blob_size: 0 2024-09-21T04:13:55.018 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:54 smithi089 ceph-mon[64076]: rocksdb: Options.blob_file_size: 268435456 2024-09-21T04:13:55.018 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:54 smithi089 ceph-mon[64076]: rocksdb: Options.blob_compression_type: NoCompression 2024-09-21T04:13:55.019 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:54 smithi089 ceph-mon[64076]: rocksdb: Options.enable_blob_garbage_collection: false 2024-09-21T04:13:55.019 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:54 smithi089 ceph-mon[64076]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-09-21T04:13:55.019 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:54 smithi089 ceph-mon[64076]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-09-21T04:13:55.019 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:54 smithi089 ceph-mon[64076]: rocksdb: Options.blob_compaction_readahead_size: 0 2024-09-21T04:13:55.019 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:54 smithi089 ceph-mon[64076]: rocksdb: Options.blob_file_starting_level: 0 2024-09-21T04:13:55.019 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:54 smithi089 ceph-mon[64076]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-09-21T04:13:55.019 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:54 smithi089 ceph-mon[64076]: rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 30.sst 2024-09-21T04:13:55.019 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:54 smithi089 ceph-mon[64076]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-09-21T04:13:55.019 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:54 smithi089 ceph-mon[64076]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-smithi089/store.db/MANIFEST-000015 succeeded,manifest_file_number is 15, next_file_number is 32, last_sequence is 11918, log_number is 28,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-09-21T04:13:55.019 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:54 smithi089 ceph-mon[64076]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 28 2024-09-21T04:13:55.019 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:54 smithi089 ceph-mon[64076]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: ec7c5a60-1c31-4106-9df1-609690964e8c 2024-09-21T04:13:55.019 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:54 smithi089 ceph-mon[64076]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726892034815910, "job": 1, "event": "recovery_started", "wal_files": [28]} 2024-09-21T04:13:55.019 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:54 smithi089 ceph-mon[64076]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #28 mode 2 2024-09-21T04:13:55.020 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:54 smithi089 ceph-mon[64076]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726892034821804, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 33, "file_size": 1452086, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 11919, "largest_seqno": 12711, "table_properties": {"data_size": 1447791, "index_size": 2359, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 901, "raw_key_size": 7901, "raw_average_key_size": 23, "raw_value_size": 1440284, "raw_average_value_size": 4248, "num_data_blocks": 108, "num_entries": 339, "num_filter_entries": 339, "num_deletions": 2, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[CompactOnDeletionCollector]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1726892034, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "ec7c5a60-1c31-4106-9df1-609690964e8c", "db_session_id": "VULAKVJBO0F1TJ74V1FE", "orig_file_number": 33, "seqno_to_time_mapping": "N/A"}} 2024-09-21T04:13:55.020 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:54 smithi089 ceph-mon[64076]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726892034821921, "job": 1, "event": "recovery_finished"} 2024-09-21T04:13:55.020 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:54 smithi089 ceph-mon[64076]: rocksdb: [db/version_set.cc:5047] Creating manifest 35 2024-09-21T04:13:55.020 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:54 smithi089 ceph-mon[64076]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-09-21T04:13:55.020 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:54 smithi089 ceph-mon[64076]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-smithi089/store.db/000028.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-09-21T04:13:55.020 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:54 smithi089 ceph-mon[64076]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x55dcee278e00 2024-09-21T04:13:55.020 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:54 smithi089 ceph-mon[64076]: rocksdb: DB pointer 0x55dcee37a000 2024-09-21T04:13:55.020 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:54 smithi089 ceph-mon[64076]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-09-21T04:13:55.020 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:54 smithi089 ceph-mon[64076]: rocksdb: [db/db_impl/db_impl.cc:1111] 2024-09-21T04:13:55.020 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout: ** DB Stats ** 2024-09-21T04:13:55.020 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-21T04:13:55.020 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-09-21T04:13:55.020 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-21T04:13:55.021 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-21T04:13:55.021 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-09-21T04:13:55.021 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-21T04:13:55.021 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-21T04:13:55.021 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout: 2024-09-21T04:13:55.021 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout: ** Compaction Stats [default] ** 2024-09-21T04:13:55.021 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-09-21T04:13:55.021 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-09-21T04:13:55.021 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout: L0 1/0 1.38 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 393.7 0.00 0.00 1 0.004 0 0 0.0 0.0 2024-09-21T04:13:55.021 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout: L6 1/0 8.01 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2024-09-21T04:13:55.021 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout: Sum 2/0 9.39 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 393.7 0.00 0.00 1 0.004 0 0 0.0 0.0 2024-09-21T04:13:55.021 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 393.7 0.00 0.00 1 0.004 0 0 0.0 0.0 2024-09-21T04:13:55.022 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout: 2024-09-21T04:13:55.022 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout: ** Compaction Stats [default] ** 2024-09-21T04:13:55.022 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-09-21T04:13:55.022 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-21T04:13:55.022 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 393.7 0.00 0.00 1 0.004 0 0 0.0 0.0 2024-09-21T04:13:55.022 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout: 2024-09-21T04:13:55.022 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-09-21T04:13:55.022 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout: 2024-09-21T04:13:55.022 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-21T04:13:55.022 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout: Flush(GB): cumulative 0.001, interval 0.001 2024-09-21T04:13:55.022 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-09-21T04:13:55.022 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-09-21T04:13:55.023 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-09-21T04:13:55.023 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout: AddFile(Keys): cumulative 0, interval 0 2024-09-21T04:13:55.023 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout: Cumulative compaction: 0.00 GB write, 130.95 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-21T04:13:55.023 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout: Interval compaction: 0.00 GB write, 130.95 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-21T04:13:55.023 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-09-21T04:13:55.023 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout: Block cache BinnedLRUCache@0x55dcee249610#2 capacity: 512.00 MB usage: 28.72 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1.2e-05 secs_since: 0 2024-09-21T04:13:55.023 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout: Block cache entry stats(count,size,portion): FilterBlock(2,10.06 KB,0.00191927%) IndexBlock(2,18.66 KB,0.0035584%) Misc(1,0.00 KB,0%) 2024-09-21T04:13:55.023 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout: 2024-09-21T04:13:55.023 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout: ** File Read Latency Histogram By Level [default] ** 2024-09-21T04:13:55.023 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:54 smithi089 ceph-mon[64076]: starting mon.smithi089 rank 0 at public addrs [v2:172.21.15.89:3300/0,v1:172.21.15.89:6789/0] at bind addrs [v2:172.21.15.89:3300/0,v1:172.21.15.89:6789/0] mon_data /var/lib/ceph/mon/ceph-smithi089 fsid 1b9b79fc-77ce-11ef-baf5-efdc52797490 2024-09-21T04:13:55.023 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:54 smithi089 ceph-mon[64076]: mon.smithi089@-1(???) e2 preinit fsid 1b9b79fc-77ce-11ef-baf5-efdc52797490 2024-09-21T04:13:55.023 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:54 smithi089 ceph-mon[64076]: mon.smithi089@-1(???).mds e6 new map 2024-09-21T04:13:55.023 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:54 smithi089 ceph-mon[64076]: mon.smithi089@-1(???).mds e6 print_map 2024-09-21T04:13:55.024 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout: e6 2024-09-21T04:13:55.024 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout: btime 1970-01-01T00:00:00:000000+0000 2024-09-21T04:13:55.024 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-09-21T04:13:55.024 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-09-21T04:13:55.024 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout: legacy client fscid: 1 2024-09-21T04:13:55.024 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout: 2024-09-21T04:13:55.024 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout: Filesystem 'foofs' (1) 2024-09-21T04:13:55.024 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout: fs_name foofs 2024-09-21T04:13:55.024 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout: epoch 4 2024-09-21T04:13:55.025 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout: flags 12 joinable allow_snaps allow_multimds_snaps 2024-09-21T04:13:55.025 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout: created 2024-09-21T04:08:42.283234+0000 2024-09-21T04:13:55.025 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout: modified 2024-09-21T04:08:48.134822+0000 2024-09-21T04:13:55.025 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout: tableserver 0 2024-09-21T04:13:55.025 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout: root 0 2024-09-21T04:13:55.025 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout: session_timeout 60 2024-09-21T04:13:55.025 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout: session_autoclose 300 2024-09-21T04:13:55.025 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout: max_file_size 1099511627776 2024-09-21T04:13:55.025 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout: max_xattr_size 65536 2024-09-21T04:13:55.025 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout: required_client_features {} 2024-09-21T04:13:55.025 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout: last_failure 0 2024-09-21T04:13:55.025 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout: last_failure_osd_epoch 0 2024-09-21T04:13:55.025 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-09-21T04:13:55.026 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout: max_mds 1 2024-09-21T04:13:55.026 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout: in 0 2024-09-21T04:13:55.026 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout: up {0=14500} 2024-09-21T04:13:55.026 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout: failed 2024-09-21T04:13:55.026 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout: damaged 2024-09-21T04:13:55.026 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout: stopped 2024-09-21T04:13:55.026 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout: data_pools [3] 2024-09-21T04:13:55.026 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout: metadata_pool 2 2024-09-21T04:13:55.026 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout: inline_data disabled 2024-09-21T04:13:55.026 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout: balancer 2024-09-21T04:13:55.026 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout: bal_rank_mask -1 2024-09-21T04:13:55.026 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout: standby_count_wanted 1 2024-09-21T04:13:55.027 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout: qdb_cluster leader: 0 members: 2024-09-21T04:13:55.027 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout: [mds.foofs.smithi089.osufws{0:14500} state up:active seq 2 join_fscid=1 addr [v2:172.21.15.89:6834/345305443,v1:172.21.15.89:6835/345305443] compat {c=[1],r=[1],i=[7ff]}] 2024-09-21T04:13:55.027 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout: 2024-09-21T04:13:55.027 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout: 2024-09-21T04:13:55.027 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout: Standby daemons: 2024-09-21T04:13:55.027 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout: 2024-09-21T04:13:55.027 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout: [mds.foofs.smithi195.hijxre{-1:24281} state up:standby seq 1 join_fscid=1 addr [v2:172.21.15.195:6832/3071482597,v1:172.21.15.195:6833/3071482597] compat {c=[1],r=[1],i=[7ff]}] 2024-09-21T04:13:55.027 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:54 smithi089 ceph-mon[64076]: mon.smithi089@-1(???).osd e73 crush map has features 3314933000854323200, adjusting msgr requires 2024-09-21T04:13:55.027 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:54 smithi089 ceph-mon[64076]: mon.smithi089@-1(???).osd e73 crush map has features 432629239337189376, adjusting msgr requires 2024-09-21T04:13:55.027 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:54 smithi089 ceph-mon[64076]: mon.smithi089@-1(???).osd e73 crush map has features 432629239337189376, adjusting msgr requires 2024-09-21T04:13:55.027 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:54 smithi089 ceph-mon[64076]: mon.smithi089@-1(???).osd e73 crush map has features 432629239337189376, adjusting msgr requires 2024-09-21T04:13:55.027 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:54 smithi089 ceph-mon[64076]: mon.smithi089@-1(???).paxosservice(auth 1..29) refresh upgraded, format 0 -> 3 2024-09-21T04:13:55.027 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:54 smithi089 bash[64057]: 7dd34c8a6f21e84b210383e6f843ee23f5d95364b9b5c161c29c2892088e3af1 2024-09-21T04:13:55.028 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:54 smithi089 systemd[1]: Started Ceph mon.smithi089 for 1b9b79fc-77ce-11ef-baf5-efdc52797490. 2024-09-21T04:13:55.256 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:55 smithi089 ceph-mon[64076]: from='mgr.14806 172.21.15.89:0/3152262325' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "mon metadata", "id": "smithi089"}]: dispatch 2024-09-21T04:13:55.256 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:55 smithi089 ceph-mon[64076]: from='mgr.14806 172.21.15.89:0/3152262325' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "mon metadata", "id": "smithi195"}]: dispatch 2024-09-21T04:13:55.256 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:55 smithi089 ceph-mon[64076]: from='mgr.14806 172.21.15.89:0/3152262325' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-21T04:13:55.256 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:55 smithi089 ceph-mon[64076]: from='mon.? -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-21T04:13:55.256 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:55 smithi089 ceph-mon[64076]: from='mgr.14806 172.21.15.89:0/3152262325' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T04:13:55.256 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:55 smithi089 ceph-mon[64076]: pgmap v26: 97 pgs: 97 active+clean; 327 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 6.1 KiB/s rd, 4.2 MiB/s wr, 413 op/s 2024-09-21T04:13:55.256 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:55 smithi089 ceph-mon[64076]: mon.smithi089 calling monitor election 2024-09-21T04:13:55.256 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:55 smithi089 ceph-mon[64076]: mon.smithi089 is new leader, mons smithi089,smithi195 in quorum (ranks 0,1) 2024-09-21T04:13:55.257 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:55 smithi089 ceph-mon[64076]: monmap epoch 2 2024-09-21T04:13:55.257 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:55 smithi089 ceph-mon[64076]: fsid 1b9b79fc-77ce-11ef-baf5-efdc52797490 2024-09-21T04:13:55.257 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:55 smithi089 ceph-mon[64076]: last_changed 2024-09-21T04:03:32.716481+0000 2024-09-21T04:13:55.257 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:55 smithi089 ceph-mon[64076]: created 2024-09-21T04:01:28.028960+0000 2024-09-21T04:13:55.257 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:55 smithi089 ceph-mon[64076]: min_mon_release 17 (quincy) 2024-09-21T04:13:55.257 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:55 smithi089 ceph-mon[64076]: election_strategy: 1 2024-09-21T04:13:55.257 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:55 smithi089 ceph-mon[64076]: 0: [v2:172.21.15.89:3300/0,v1:172.21.15.89:6789/0] mon.smithi089 2024-09-21T04:13:55.257 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:55 smithi089 ceph-mon[64076]: 1: [v2:172.21.15.195:3300/0,v1:172.21.15.195:6789/0] mon.smithi195 2024-09-21T04:13:55.257 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:55 smithi089 ceph-mon[64076]: fsmap foofs:1 {0=foofs.smithi089.osufws=up:active} 1 up:standby 2024-09-21T04:13:55.257 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:55 smithi089 ceph-mon[64076]: osdmap e73: 8 total, 8 up, 8 in 2024-09-21T04:13:55.257 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:55 smithi089 ceph-mon[64076]: mgrmap e36: smithi089.srlyxf(active, since 48s), standbys: smithi195.cgotzm 2024-09-21T04:13:55.257 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:55 smithi089 ceph-mon[64076]: overall HEALTH_OK 2024-09-21T04:13:55.257 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:55 smithi089 ceph-mon[64076]: from='mgr.14806 ' entity='' 2024-09-21T04:13:55.257 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:55 smithi089 ceph-mon[64076]: mgrmap e37: smithi089.srlyxf(active, since 48s), standbys: smithi195.cgotzm 2024-09-21T04:13:55.336 INFO:teuthology.orchestra.run.smithi089.stdout: 5 124781 10.53 MB/sec execute 154 sec latency 35.849 ms 2024-09-21T04:13:55.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:13:55 smithi195 ceph-mon[25840]: from='mgr.14806 172.21.15.89:0/3152262325' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "mon metadata", "id": "smithi089"}]: dispatch 2024-09-21T04:13:55.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:13:55 smithi195 ceph-mon[25840]: from='mgr.14806 172.21.15.89:0/3152262325' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "mon metadata", "id": "smithi195"}]: dispatch 2024-09-21T04:13:55.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:13:55 smithi195 ceph-mon[25840]: from='mgr.14806 172.21.15.89:0/3152262325' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-21T04:13:55.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:13:55 smithi195 ceph-mon[25840]: from='mon.? -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-21T04:13:55.360 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:13:55 smithi195 ceph-mon[25840]: from='mgr.14806 172.21.15.89:0/3152262325' entity='mgr.smithi089.srlyxf' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T04:13:55.360 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:13:55 smithi195 ceph-mon[25840]: pgmap v26: 97 pgs: 97 active+clean; 327 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 6.1 KiB/s rd, 4.2 MiB/s wr, 413 op/s 2024-09-21T04:13:55.360 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:13:55 smithi195 ceph-mon[25840]: mon.smithi089 calling monitor election 2024-09-21T04:13:55.360 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:13:55 smithi195 ceph-mon[25840]: mon.smithi089 is new leader, mons smithi089,smithi195 in quorum (ranks 0,1) 2024-09-21T04:13:55.360 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:13:55 smithi195 ceph-mon[25840]: monmap epoch 2 2024-09-21T04:13:55.360 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:13:55 smithi195 ceph-mon[25840]: fsid 1b9b79fc-77ce-11ef-baf5-efdc52797490 2024-09-21T04:13:55.360 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:13:55 smithi195 ceph-mon[25840]: last_changed 2024-09-21T04:03:32.716481+0000 2024-09-21T04:13:55.360 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:13:55 smithi195 ceph-mon[25840]: created 2024-09-21T04:01:28.028960+0000 2024-09-21T04:13:55.360 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:13:55 smithi195 ceph-mon[25840]: min_mon_release 17 (quincy) 2024-09-21T04:13:55.360 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:13:55 smithi195 ceph-mon[25840]: election_strategy: 1 2024-09-21T04:13:55.360 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:13:55 smithi195 ceph-mon[25840]: 0: [v2:172.21.15.89:3300/0,v1:172.21.15.89:6789/0] mon.smithi089 2024-09-21T04:13:55.360 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:13:55 smithi195 ceph-mon[25840]: 1: [v2:172.21.15.195:3300/0,v1:172.21.15.195:6789/0] mon.smithi195 2024-09-21T04:13:55.360 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:13:55 smithi195 ceph-mon[25840]: fsmap foofs:1 {0=foofs.smithi089.osufws=up:active} 1 up:standby 2024-09-21T04:13:55.360 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:13:55 smithi195 ceph-mon[25840]: osdmap e73: 8 total, 8 up, 8 in 2024-09-21T04:13:55.360 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:13:55 smithi195 ceph-mon[25840]: mgrmap e36: smithi089.srlyxf(active, since 48s), standbys: smithi195.cgotzm 2024-09-21T04:13:55.361 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:13:55 smithi195 ceph-mon[25840]: overall HEALTH_OK 2024-09-21T04:13:55.361 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:13:55 smithi195 ceph-mon[25840]: from='mgr.14806 ' entity='' 2024-09-21T04:13:55.361 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:13:55 smithi195 ceph-mon[25840]: mgrmap e37: smithi089.srlyxf(active, since 48s), standbys: smithi195.cgotzm 2024-09-21T04:13:56.337 INFO:teuthology.orchestra.run.smithi089.stdout: 5 125052 10.48 MB/sec execute 155 sec latency 73.908 ms 2024-09-21T04:13:57.068 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:13:57 smithi089 ceph-mon[64076]: mgrmap e38: smithi089.srlyxf(active, since 49s), standbys: smithi195.cgotzm 2024-09-21T04:13:57.338 INFO:teuthology.orchestra.run.smithi089.stdout: 5 125279 10.45 MB/sec execute 156 sec latency 55.371 ms 2024-09-21T04:13:57.358 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:13:57 smithi195 ceph-mon[25840]: mgrmap e38: smithi089.srlyxf(active, since 49s), standbys: smithi195.cgotzm 2024-09-21T04:13:58.337 INFO:teuthology.orchestra.run.smithi089.stdout: 5 125533 10.41 MB/sec execute 157 sec latency 70.767 ms 2024-09-21T04:13:59.337 INFO:teuthology.orchestra.run.smithi089.stdout: 5 125894 10.43 MB/sec execute 158 sec latency 76.135 ms 2024-09-21T04:14:00.337 INFO:teuthology.orchestra.run.smithi089.stdout: 5 126275 10.38 MB/sec execute 159 sec latency 86.319 ms 2024-09-21T04:14:01.337 INFO:teuthology.orchestra.run.smithi089.stdout: 5 126646 10.37 MB/sec execute 160 sec latency 36.708 ms 2024-09-21T04:14:01.497 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:14:01 smithi089 ceph-mon[64076]: Standby manager daemon smithi195.cgotzm restarted 2024-09-21T04:14:01.497 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:14:01 smithi089 ceph-mon[64076]: Standby manager daemon smithi195.cgotzm started 2024-09-21T04:14:01.497 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:14:01 smithi089 ceph-mon[64076]: from='mgr.? 172.21.15.195:0/1690933579' entity='mgr.smithi195.cgotzm' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi195.cgotzm/crt"}]: dispatch 2024-09-21T04:14:01.497 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:14:01 smithi089 ceph-mon[64076]: from='mgr.? 172.21.15.195:0/1690933579' entity='mgr.smithi195.cgotzm' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-21T04:14:01.497 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:14:01 smithi089 ceph-mon[64076]: from='mgr.? 172.21.15.195:0/1690933579' entity='mgr.smithi195.cgotzm' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi195.cgotzm/key"}]: dispatch 2024-09-21T04:14:01.497 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:14:01 smithi089 ceph-mon[64076]: from='mgr.? 172.21.15.195:0/1690933579' entity='mgr.smithi195.cgotzm' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-21T04:14:01.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:01 smithi195 ceph-mon[25840]: Standby manager daemon smithi195.cgotzm restarted 2024-09-21T04:14:01.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:01 smithi195 ceph-mon[25840]: Standby manager daemon smithi195.cgotzm started 2024-09-21T04:14:01.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:01 smithi195 ceph-mon[25840]: from='mgr.? 172.21.15.195:0/1690933579' entity='mgr.smithi195.cgotzm' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi195.cgotzm/crt"}]: dispatch 2024-09-21T04:14:01.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:01 smithi195 ceph-mon[25840]: from='mgr.? 172.21.15.195:0/1690933579' entity='mgr.smithi195.cgotzm' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-21T04:14:01.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:01 smithi195 ceph-mon[25840]: from='mgr.? 172.21.15.195:0/1690933579' entity='mgr.smithi195.cgotzm' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi195.cgotzm/key"}]: dispatch 2024-09-21T04:14:01.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:01 smithi195 ceph-mon[25840]: from='mgr.? 172.21.15.195:0/1690933579' entity='mgr.smithi195.cgotzm' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-21T04:14:02.337 INFO:teuthology.orchestra.run.smithi089.stdout: 5 126863 10.34 MB/sec execute 161 sec latency 67.668 ms 2024-09-21T04:14:02.515 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:14:02 smithi089 ceph-mon[64076]: mgrmap e39: smithi089.srlyxf(active, since 54s), standbys: smithi195.cgotzm 2024-09-21T04:14:02.515 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:14:02 smithi089 ceph-mon[64076]: Active manager daemon smithi089.srlyxf restarted 2024-09-21T04:14:02.515 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:14:02 smithi089 ceph-mon[64076]: Activating manager daemon smithi089.srlyxf 2024-09-21T04:14:02.515 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:14:02 smithi089 ceph-mon[64076]: osdmap e74: 8 total, 8 up, 8 in 2024-09-21T04:14:02.515 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:14:02 smithi089 ceph-mon[64076]: mgrmap e40: smithi089.srlyxf(active, starting, since 0.00892451s), standbys: smithi195.cgotzm 2024-09-21T04:14:02.515 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:14:02 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "mon metadata", "id": "smithi089"} : dispatch 2024-09-21T04:14:02.515 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:14:02 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "mon metadata", "id": "smithi195"} : dispatch 2024-09-21T04:14:02.515 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:14:02 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "mds metadata", "who": "foofs.smithi089.osufws"} : dispatch 2024-09-21T04:14:02.516 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:14:02 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "mds metadata", "who": "foofs.smithi195.hijxre"} : dispatch 2024-09-21T04:14:02.516 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:14:02 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "mgr metadata", "who": "smithi089.srlyxf", "id": "smithi089.srlyxf"} : dispatch 2024-09-21T04:14:02.516 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:14:02 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "mgr metadata", "who": "smithi195.cgotzm", "id": "smithi195.cgotzm"} : dispatch 2024-09-21T04:14:02.516 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:14:02 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "mds metadata"} : dispatch 2024-09-21T04:14:02.516 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:14:02 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "osd metadata"} : dispatch 2024-09-21T04:14:02.516 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:14:02 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "mon metadata"} : dispatch 2024-09-21T04:14:02.516 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:14:02 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "osd metadata", "id": 0} : dispatch 2024-09-21T04:14:02.516 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:14:02 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "osd metadata", "id": 1} : dispatch 2024-09-21T04:14:02.516 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:14:02 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "osd metadata", "id": 2} : dispatch 2024-09-21T04:14:02.516 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:14:02 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "osd metadata", "id": 3} : dispatch 2024-09-21T04:14:02.516 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:14:02 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "osd metadata", "id": 4} : dispatch 2024-09-21T04:14:02.516 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:14:02 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "osd metadata", "id": 5} : dispatch 2024-09-21T04:14:02.516 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:14:02 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "osd metadata", "id": 6} : dispatch 2024-09-21T04:14:02.516 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:14:02 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "osd metadata", "id": 7} : dispatch 2024-09-21T04:14:02.516 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:14:02 smithi089 ceph-mon[64076]: Manager daemon smithi089.srlyxf is now available 2024-09-21T04:14:02.517 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:14:02 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T04:14:02.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:02 smithi195 ceph-mon[25840]: mgrmap e39: smithi089.srlyxf(active, since 54s), standbys: smithi195.cgotzm 2024-09-21T04:14:02.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:02 smithi195 ceph-mon[25840]: Active manager daemon smithi089.srlyxf restarted 2024-09-21T04:14:02.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:02 smithi195 ceph-mon[25840]: Activating manager daemon smithi089.srlyxf 2024-09-21T04:14:02.610 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:02 smithi195 ceph-mon[25840]: osdmap e74: 8 total, 8 up, 8 in 2024-09-21T04:14:02.610 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:02 smithi195 ceph-mon[25840]: mgrmap e40: smithi089.srlyxf(active, starting, since 0.00892451s), standbys: smithi195.cgotzm 2024-09-21T04:14:02.610 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:02 smithi195 ceph-mon[25840]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "mon metadata", "id": "smithi089"} : dispatch 2024-09-21T04:14:02.610 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:02 smithi195 ceph-mon[25840]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "mon metadata", "id": "smithi195"} : dispatch 2024-09-21T04:14:02.610 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:02 smithi195 ceph-mon[25840]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "mds metadata", "who": "foofs.smithi089.osufws"} : dispatch 2024-09-21T04:14:02.610 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:02 smithi195 ceph-mon[25840]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "mds metadata", "who": "foofs.smithi195.hijxre"} : dispatch 2024-09-21T04:14:02.610 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:02 smithi195 ceph-mon[25840]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "mgr metadata", "who": "smithi089.srlyxf", "id": "smithi089.srlyxf"} : dispatch 2024-09-21T04:14:02.610 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:02 smithi195 ceph-mon[25840]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "mgr metadata", "who": "smithi195.cgotzm", "id": "smithi195.cgotzm"} : dispatch 2024-09-21T04:14:02.610 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:02 smithi195 ceph-mon[25840]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "mds metadata"} : dispatch 2024-09-21T04:14:02.610 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:02 smithi195 ceph-mon[25840]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "osd metadata"} : dispatch 2024-09-21T04:14:02.610 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:02 smithi195 ceph-mon[25840]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "mon metadata"} : dispatch 2024-09-21T04:14:02.611 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:02 smithi195 ceph-mon[25840]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "osd metadata", "id": 0} : dispatch 2024-09-21T04:14:02.611 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:02 smithi195 ceph-mon[25840]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "osd metadata", "id": 1} : dispatch 2024-09-21T04:14:02.611 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:02 smithi195 ceph-mon[25840]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "osd metadata", "id": 2} : dispatch 2024-09-21T04:14:02.611 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:02 smithi195 ceph-mon[25840]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "osd metadata", "id": 3} : dispatch 2024-09-21T04:14:02.611 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:02 smithi195 ceph-mon[25840]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "osd metadata", "id": 4} : dispatch 2024-09-21T04:14:02.611 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:02 smithi195 ceph-mon[25840]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "osd metadata", "id": 5} : dispatch 2024-09-21T04:14:02.611 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:02 smithi195 ceph-mon[25840]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "osd metadata", "id": 6} : dispatch 2024-09-21T04:14:02.611 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:02 smithi195 ceph-mon[25840]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "osd metadata", "id": 7} : dispatch 2024-09-21T04:14:02.611 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:02 smithi195 ceph-mon[25840]: Manager daemon smithi089.srlyxf is now available 2024-09-21T04:14:02.611 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:02 smithi195 ceph-mon[25840]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T04:14:03.337 INFO:teuthology.orchestra.run.smithi089.stdout: 5 127231 10.41 MB/sec execute 162 sec latency 47.864 ms 2024-09-21T04:14:03.534 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:14:03 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "orch prometheus get-credentials"} : dispatch 2024-09-21T04:14:03.534 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:14:03 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T04:14:03.534 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:14:03 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi089.srlyxf/mirror_snapshot_schedule"} : dispatch 2024-09-21T04:14:03.534 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:14:03 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi089.srlyxf/trash_purge_schedule"} : dispatch 2024-09-21T04:14:03.534 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:14:03 smithi089 ceph-mon[64076]: mgrmap e41: smithi089.srlyxf(active, since 1.01612s), standbys: smithi195.cgotzm 2024-09-21T04:14:03.534 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:14:03 smithi089 ceph-mon[64076]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-21T04:14:03.534 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:14:03 smithi089 ceph-mon[64076]: pgmap v3: 97 pgs: 97 active+clean; 328 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-09-21T04:14:03.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:03 smithi195 ceph-mon[25840]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "orch prometheus get-credentials"} : dispatch 2024-09-21T04:14:03.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:03 smithi195 ceph-mon[25840]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T04:14:03.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:03 smithi195 ceph-mon[25840]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi089.srlyxf/mirror_snapshot_schedule"} : dispatch 2024-09-21T04:14:03.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:03 smithi195 ceph-mon[25840]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi089.srlyxf/trash_purge_schedule"} : dispatch 2024-09-21T04:14:03.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:03 smithi195 ceph-mon[25840]: mgrmap e41: smithi089.srlyxf(active, since 1.01612s), standbys: smithi195.cgotzm 2024-09-21T04:14:03.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:03 smithi195 ceph-mon[25840]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-21T04:14:03.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:03 smithi195 ceph-mon[25840]: pgmap v3: 97 pgs: 97 active+clean; 328 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-09-21T04:14:04.337 INFO:teuthology.orchestra.run.smithi089.stdout: 5 127632 10.42 MB/sec execute 163 sec latency 32.395 ms 2024-09-21T04:14:05.100 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:14:04 smithi089 ceph-mon[64076]: [21/Sep/2024:04:14:03] ENGINE Bus STARTING 2024-09-21T04:14:05.100 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:14:04 smithi089 ceph-mon[64076]: [21/Sep/2024:04:14:03] ENGINE Serving on http://172.21.15.89:8765 2024-09-21T04:14:05.100 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:14:04 smithi089 ceph-mon[64076]: pgmap v4: 97 pgs: 97 active+clean; 328 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-09-21T04:14:05.100 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:14:04 smithi089 ceph-mon[64076]: [21/Sep/2024:04:14:03] ENGINE Serving on https://172.21.15.89:7150 2024-09-21T04:14:05.100 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:14:04 smithi089 ceph-mon[64076]: [21/Sep/2024:04:14:03] ENGINE Bus STARTED 2024-09-21T04:14:05.100 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:14:04 smithi089 ceph-mon[64076]: [21/Sep/2024:04:14:03] ENGINE Error in HTTPServer.serve 2024-09-21T04:14:05.100 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout: Traceback (most recent call last): 2024-09-21T04:14:05.100 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-21T04:14:05.100 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout: self._connections.run(self.expiration_interval) 2024-09-21T04:14:05.100 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-21T04:14:05.100 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout: self._run(expiration_interval) 2024-09-21T04:14:05.101 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-21T04:14:05.101 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-21T04:14:05.101 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-21T04:14:05.101 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-21T04:14:05.101 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-21T04:14:05.101 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout: s = self.context.wrap_socket( 2024-09-21T04:14:05.101 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-21T04:14:05.101 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout: return self.sslsocket_class._create( 2024-09-21T04:14:05.101 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-21T04:14:05.101 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout: self.do_handshake() 2024-09-21T04:14:05.101 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-21T04:14:05.101 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout: self._sslobj.do_handshake() 2024-09-21T04:14:05.101 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-21T04:14:05.101 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:14:04 smithi089 ceph-mon[64076]: mgrmap e42: smithi089.srlyxf(active, since 2s), standbys: smithi195.cgotzm 2024-09-21T04:14:05.217 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:04 smithi195 ceph-mon[25840]: [21/Sep/2024:04:14:03] ENGINE Bus STARTING 2024-09-21T04:14:05.218 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:04 smithi195 ceph-mon[25840]: [21/Sep/2024:04:14:03] ENGINE Serving on http://172.21.15.89:8765 2024-09-21T04:14:05.218 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:04 smithi195 ceph-mon[25840]: pgmap v4: 97 pgs: 97 active+clean; 328 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-09-21T04:14:05.218 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:04 smithi195 ceph-mon[25840]: [21/Sep/2024:04:14:03] ENGINE Serving on https://172.21.15.89:7150 2024-09-21T04:14:05.218 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:04 smithi195 ceph-mon[25840]: [21/Sep/2024:04:14:03] ENGINE Bus STARTED 2024-09-21T04:14:05.218 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:04 smithi195 ceph-mon[25840]: [21/Sep/2024:04:14:03] ENGINE Error in HTTPServer.serve 2024-09-21T04:14:05.218 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout: Traceback (most recent call last): 2024-09-21T04:14:05.218 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-21T04:14:05.218 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout: self._connections.run(self.expiration_interval) 2024-09-21T04:14:05.218 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-21T04:14:05.218 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout: self._run(expiration_interval) 2024-09-21T04:14:05.218 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-21T04:14:05.218 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-21T04:14:05.218 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-21T04:14:05.219 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-21T04:14:05.219 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-21T04:14:05.219 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout: s = self.context.wrap_socket( 2024-09-21T04:14:05.219 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-21T04:14:05.219 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout: return self.sslsocket_class._create( 2024-09-21T04:14:05.219 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-21T04:14:05.219 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout: self.do_handshake() 2024-09-21T04:14:05.219 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-21T04:14:05.219 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout: self._sslobj.do_handshake() 2024-09-21T04:14:05.219 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-21T04:14:05.219 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:04 smithi195 ceph-mon[25840]: mgrmap e42: smithi089.srlyxf(active, since 2s), standbys: smithi195.cgotzm 2024-09-21T04:14:05.337 INFO:teuthology.orchestra.run.smithi089.stdout: 5 127915 10.44 MB/sec execute 164 sec latency 61.296 ms 2024-09-21T04:14:06.337 INFO:teuthology.orchestra.run.smithi089.stdout: 5 128165 10.40 MB/sec execute 165 sec latency 56.189 ms 2024-09-21T04:14:07.134 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:14:06 smithi089 ceph-mon[64076]: pgmap v5: 97 pgs: 97 active+clean; 328 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-09-21T04:14:07.134 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:14:06 smithi089 ceph-mon[64076]: mgrmap e43: smithi089.srlyxf(active, since 4s), standbys: smithi195.cgotzm 2024-09-21T04:14:07.134 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:14:06 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:14:07.134 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:14:06 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:14:07.337 INFO:teuthology.orchestra.run.smithi089.stdout: 5 128467 10.35 MB/sec execute 166 sec latency 63.058 ms 2024-09-21T04:14:07.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:06 smithi195 ceph-mon[25840]: pgmap v5: 97 pgs: 97 active+clean; 328 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-09-21T04:14:07.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:06 smithi195 ceph-mon[25840]: mgrmap e43: smithi089.srlyxf(active, since 4s), standbys: smithi195.cgotzm 2024-09-21T04:14:07.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:06 smithi195 ceph-mon[25840]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:14:07.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:06 smithi195 ceph-mon[25840]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:14:08.149 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:14:08 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:14:08.149 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:14:08 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:14:08.150 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:14:08 smithi089 ceph-mon[64076]: pgmap v6: 97 pgs: 97 active+clean; 328 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-09-21T04:14:08.337 INFO:teuthology.orchestra.run.smithi089.stdout: 5 128799 10.33 MB/sec execute 167 sec latency 50.418 ms 2024-09-21T04:14:08.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:08 smithi195 ceph-mon[25840]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:14:08.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:08 smithi195 ceph-mon[25840]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:14:08.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:08 smithi195 ceph-mon[25840]: pgmap v6: 97 pgs: 97 active+clean; 328 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-09-21T04:14:09.337 INFO:teuthology.orchestra.run.smithi089.stdout: 5 129071 10.30 MB/sec execute 168 sec latency 56.055 ms 2024-09-21T04:14:10.338 INFO:teuthology.orchestra.run.smithi089.stdout: 5 129427 10.32 MB/sec execute 169 sec latency 66.627 ms 2024-09-21T04:14:11.014 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:14:10 smithi089 ceph-mon[64076]: pgmap v7: 97 pgs: 97 active+clean; 339 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 30 KiB/s rd, 3.9 MiB/s wr, 346 op/s 2024-09-21T04:14:11.015 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:14:10 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:14:11.015 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:14:10 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:14:11.109 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:10 smithi195 ceph-mon[25840]: pgmap v7: 97 pgs: 97 active+clean; 339 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 30 KiB/s rd, 3.9 MiB/s wr, 346 op/s 2024-09-21T04:14:11.109 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:10 smithi195 ceph-mon[25840]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:14:11.109 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:10 smithi195 ceph-mon[25840]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:14:11.338 INFO:teuthology.orchestra.run.smithi089.stdout: 5 129818 10.27 MB/sec execute 170 sec latency 88.044 ms 2024-09-21T04:14:12.338 INFO:teuthology.orchestra.run.smithi089.stdout: 5 130191 10.26 MB/sec execute 171 sec latency 71.896 ms 2024-09-21T04:14:12.549 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:14:12 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:14:12.549 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:14:12 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:14:12.549 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:14:12 smithi089 ceph-mon[64076]: pgmap v8: 97 pgs: 97 active+clean; 339 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 23 KiB/s rd, 3.0 MiB/s wr, 268 op/s 2024-09-21T04:14:12.549 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:14:12 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:14:12.549 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:14:12 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:14:12.549 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:14:12 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T04:14:12.549 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:14:12 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T04:14:12.549 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:14:12 smithi089 ceph-mon[64076]: Updating smithi089:/etc/ceph/ceph.conf 2024-09-21T04:14:12.549 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:14:12 smithi089 ceph-mon[64076]: Updating smithi195:/etc/ceph/ceph.conf 2024-09-21T04:14:12.859 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:12 smithi195 ceph-mon[25840]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:14:12.859 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:12 smithi195 ceph-mon[25840]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:14:12.859 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:12 smithi195 ceph-mon[25840]: pgmap v8: 97 pgs: 97 active+clean; 339 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 23 KiB/s rd, 3.0 MiB/s wr, 268 op/s 2024-09-21T04:14:12.859 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:12 smithi195 ceph-mon[25840]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:14:12.859 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:12 smithi195 ceph-mon[25840]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:14:12.859 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:12 smithi195 ceph-mon[25840]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T04:14:12.859 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:12 smithi195 ceph-mon[25840]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T04:14:12.859 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:12 smithi195 ceph-mon[25840]: Updating smithi089:/etc/ceph/ceph.conf 2024-09-21T04:14:12.859 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:12 smithi195 ceph-mon[25840]: Updating smithi195:/etc/ceph/ceph.conf 2024-09-21T04:14:13.338 INFO:teuthology.orchestra.run.smithi089.stdout: 5 130621 10.31 MB/sec execute 172 sec latency 31.116 ms 2024-09-21T04:14:13.821 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:14:13 smithi089 ceph-mon[64076]: Updating smithi089:/var/lib/ceph/1b9b79fc-77ce-11ef-baf5-efdc52797490/config/ceph.conf 2024-09-21T04:14:13.821 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:14:13 smithi089 ceph-mon[64076]: Updating smithi195:/var/lib/ceph/1b9b79fc-77ce-11ef-baf5-efdc52797490/config/ceph.conf 2024-09-21T04:14:13.821 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:14:13 smithi089 ceph-mon[64076]: Updating smithi089:/etc/ceph/ceph.client.admin.keyring 2024-09-21T04:14:13.821 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:14:13 smithi089 ceph-mon[64076]: Updating smithi195:/etc/ceph/ceph.client.admin.keyring 2024-09-21T04:14:13.821 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:14:13 smithi089 ceph-mon[64076]: Updating smithi195:/var/lib/ceph/1b9b79fc-77ce-11ef-baf5-efdc52797490/config/ceph.client.admin.keyring 2024-09-21T04:14:13.821 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:14:13 smithi089 ceph-mon[64076]: Updating smithi089:/var/lib/ceph/1b9b79fc-77ce-11ef-baf5-efdc52797490/config/ceph.client.admin.keyring 2024-09-21T04:14:13.821 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:14:13 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:14:13.821 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:14:13 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:14:13.821 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:14:13 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:14:13.821 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:14:13 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:14:13.822 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:14:13 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:14:13.822 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:14:13 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:14:13.822 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:14:13 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T04:14:13.822 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:14:13 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "versions"} : dispatch 2024-09-21T04:14:13.822 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:14:13 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "quorum_status"} : dispatch 2024-09-21T04:14:13.830 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:13 smithi195 ceph-mon[25840]: Updating smithi089:/var/lib/ceph/1b9b79fc-77ce-11ef-baf5-efdc52797490/config/ceph.conf 2024-09-21T04:14:13.830 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:13 smithi195 ceph-mon[25840]: Updating smithi195:/var/lib/ceph/1b9b79fc-77ce-11ef-baf5-efdc52797490/config/ceph.conf 2024-09-21T04:14:13.830 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:13 smithi195 ceph-mon[25840]: Updating smithi089:/etc/ceph/ceph.client.admin.keyring 2024-09-21T04:14:13.830 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:13 smithi195 ceph-mon[25840]: Updating smithi195:/etc/ceph/ceph.client.admin.keyring 2024-09-21T04:14:13.830 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:13 smithi195 ceph-mon[25840]: Updating smithi195:/var/lib/ceph/1b9b79fc-77ce-11ef-baf5-efdc52797490/config/ceph.client.admin.keyring 2024-09-21T04:14:13.830 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:13 smithi195 ceph-mon[25840]: Updating smithi089:/var/lib/ceph/1b9b79fc-77ce-11ef-baf5-efdc52797490/config/ceph.client.admin.keyring 2024-09-21T04:14:13.830 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:13 smithi195 ceph-mon[25840]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:14:13.830 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:13 smithi195 ceph-mon[25840]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:14:13.830 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:13 smithi195 ceph-mon[25840]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:14:13.830 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:13 smithi195 ceph-mon[25840]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:14:13.831 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:13 smithi195 ceph-mon[25840]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:14:13.831 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:13 smithi195 ceph-mon[25840]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:14:13.831 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:13 smithi195 ceph-mon[25840]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T04:14:13.831 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:13 smithi195 ceph-mon[25840]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "versions"} : dispatch 2024-09-21T04:14:13.831 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:13 smithi195 ceph-mon[25840]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "quorum_status"} : dispatch 2024-09-21T04:14:14.338 INFO:teuthology.orchestra.run.smithi089.stdout: 5 131082 10.35 MB/sec execute 173 sec latency 48.734 ms 2024-09-21T04:14:14.344 INFO:teuthology.orchestra.run.smithi089.stdout:true 2024-09-21T04:14:14.841 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:14:14 smithi089 ceph-mon[64076]: pgmap v9: 97 pgs: 97 active+clean; 330 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 23 KiB/s rd, 4.4 MiB/s wr, 454 op/s 2024-09-21T04:14:15.109 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:14 smithi195 ceph-mon[25840]: pgmap v9: 97 pgs: 97 active+clean; 330 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 23 KiB/s rd, 4.4 MiB/s wr, 454 op/s 2024-09-21T04:14:15.338 INFO:teuthology.orchestra.run.smithi089.stdout: 5 131504 10.40 MB/sec execute 174 sec latency 59.391 ms 2024-09-21T04:14:15.353 INFO:teuthology.orchestra.run.smithi089.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-21T04:14:15.353 INFO:teuthology.orchestra.run.smithi089.stdout:alertmanager.smithi089 smithi089 *:9093,9094 running (2m) 3s ago 11m 19.5M - 0.27.0 11f11916f8cd 65dfcbcafee1 2024-09-21T04:14:15.353 INFO:teuthology.orchestra.run.smithi089.stdout:crash.smithi089 smithi089 running (11m) 3s ago 11m 6409k - 17.2.0 e1d6a67b021e d1e0a8969f78 2024-09-21T04:14:15.353 INFO:teuthology.orchestra.run.smithi089.stdout:crash.smithi195 smithi195 running (10m) 8s ago 10m 6400k - 17.2.0 e1d6a67b021e 7e213ff76084 2024-09-21T04:14:15.354 INFO:teuthology.orchestra.run.smithi089.stdout:grafana.smithi089 smithi089 *:3000 running (2m) 3s ago 11m 45.0M - dad864ee21e9 ec4d6738bce7 2024-09-21T04:14:15.354 INFO:teuthology.orchestra.run.smithi089.stdout:mds.foofs.smithi089.osufws smithi089 running (5m) 3s ago 5m 37.9M - 17.2.0 e1d6a67b021e 664d41859bea 2024-09-21T04:14:15.354 INFO:teuthology.orchestra.run.smithi089.stdout:mds.foofs.smithi195.hijxre smithi195 running (5m) 8s ago 5m 16.1M - 17.2.0 e1d6a67b021e 7bc0c6542d12 2024-09-21T04:14:15.354 INFO:teuthology.orchestra.run.smithi089.stdout:mgr.smithi089.srlyxf smithi089 *:8443,9283,8765 running (93s) 3s ago 12m 489M - 19.3.0-5103-gf36283e7 48ed0a2cf89d 3cbeaaa88241 2024-09-21T04:14:15.354 INFO:teuthology.orchestra.run.smithi089.stdout:mgr.smithi195.cgotzm smithi195 *:8443,9283,8765 running (38s) 8s ago 10m 452M - 19.3.0-5103-gf36283e7 48ed0a2cf89d d3734574cf66 2024-09-21T04:14:15.354 INFO:teuthology.orchestra.run.smithi089.stdout:mon.smithi089 smithi089 running (21s) 3s ago 12m 31.9M 2048M 19.3.0-5103-gf36283e7 48ed0a2cf89d 7dd34c8a6f21 2024-09-21T04:14:15.354 INFO:teuthology.orchestra.run.smithi089.stdout:mon.smithi195 smithi195 running (10m) 8s ago 10m 51.2M 2048M 17.2.0 e1d6a67b021e 7f86c5b5d21e 2024-09-21T04:14:15.354 INFO:teuthology.orchestra.run.smithi089.stdout:nfs.foo.0.0.smithi089.dcjmqc smithi089 *:2049 running (5m) 3s ago 5m 274M - 4.0 e1d6a67b021e 98550f94e773 2024-09-21T04:14:15.354 INFO:teuthology.orchestra.run.smithi089.stdout:nfs.foo.1.0.smithi195.vhoayp smithi195 *:2049 running (5m) 8s ago 5m 87.3M - 4.0 e1d6a67b021e c60a4040130a 2024-09-21T04:14:15.354 INFO:teuthology.orchestra.run.smithi089.stdout:node-exporter.smithi089 smithi089 *:9100 running (2m) 3s ago 11m 9.96M - 1.7.0 72c9c2088986 7ef0d1588475 2024-09-21T04:14:15.354 INFO:teuthology.orchestra.run.smithi089.stdout:node-exporter.smithi195 smithi195 *:9100 running (102s) 8s ago 10m 11.2M - 1.7.0 72c9c2088986 61eafd61bc6c 2024-09-21T04:14:15.354 INFO:teuthology.orchestra.run.smithi089.stdout:osd.0 smithi089 running (10m) 3s ago 10m 472M 1935M 17.2.0 e1d6a67b021e db7c0e5020e7 2024-09-21T04:14:15.354 INFO:teuthology.orchestra.run.smithi089.stdout:osd.1 smithi089 running (9m) 3s ago 9m 397M 1935M 17.2.0 e1d6a67b021e f9ec697d08d0 2024-09-21T04:14:15.354 INFO:teuthology.orchestra.run.smithi089.stdout:osd.2 smithi089 running (9m) 3s ago 9m 326M 1935M 17.2.0 e1d6a67b021e cfa000948859 2024-09-21T04:14:15.355 INFO:teuthology.orchestra.run.smithi089.stdout:osd.3 smithi089 running (8m) 3s ago 8m 427M 1935M 17.2.0 e1d6a67b021e 31c610ed5815 2024-09-21T04:14:15.355 INFO:teuthology.orchestra.run.smithi089.stdout:osd.4 smithi195 running (8m) 8s ago 8m 442M 2704M 17.2.0 e1d6a67b021e f00e18a1f220 2024-09-21T04:14:15.355 INFO:teuthology.orchestra.run.smithi089.stdout:osd.5 smithi195 running (7m) 8s ago 7m 481M 2704M 17.2.0 e1d6a67b021e cecfd6ff1eee 2024-09-21T04:14:15.355 INFO:teuthology.orchestra.run.smithi089.stdout:osd.6 smithi195 running (7m) 8s ago 7m 411M 2704M 17.2.0 e1d6a67b021e 0d7db362cc23 2024-09-21T04:14:15.355 INFO:teuthology.orchestra.run.smithi089.stdout:osd.7 smithi195 running (7m) 8s ago 7m 538M 2704M 17.2.0 e1d6a67b021e 0109b3fea188 2024-09-21T04:14:15.355 INFO:teuthology.orchestra.run.smithi089.stdout:prometheus.smithi089 smithi089 *:9095 running (44s) 3s ago 10m 46.1M - 2.51.0 1d3b7f56885b 8c9f309a6459 2024-09-21T04:14:15.792 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:14:15 smithi089 ceph-mon[64076]: from='client.34132 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T04:14:15.793 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:14:15 smithi089 ceph-mon[64076]: from='client.34136 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T04:14:15.793 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:14:15 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:14:15.793 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:14:15 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-09-21T04:14:15.793 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:14:15 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-09-21T04:14:15.793 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:14:15 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T04:14:15.859 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:15 smithi195 ceph-mon[25840]: from='client.34132 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T04:14:15.859 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:15 smithi195 ceph-mon[25840]: from='client.34136 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T04:14:15.859 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:15 smithi195 ceph-mon[25840]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:14:15.859 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:15 smithi195 ceph-mon[25840]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-09-21T04:14:15.860 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:15 smithi195 ceph-mon[25840]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-09-21T04:14:15.860 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:15 smithi195 ceph-mon[25840]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T04:14:15.965 INFO:teuthology.orchestra.run.smithi089.stdout:{ 2024-09-21T04:14:15.965 INFO:teuthology.orchestra.run.smithi089.stdout: "mon": { 2024-09-21T04:14:15.965 INFO:teuthology.orchestra.run.smithi089.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 1, 2024-09-21T04:14:15.965 INFO:teuthology.orchestra.run.smithi089.stdout: "ceph version 19.3.0-5103-gf36283e7 (f36283e7c91cac1c01cb9a7c4ff21bd697dbd4f2) squid (dev)": 1 2024-09-21T04:14:15.965 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2024-09-21T04:14:15.966 INFO:teuthology.orchestra.run.smithi089.stdout: "mgr": { 2024-09-21T04:14:15.966 INFO:teuthology.orchestra.run.smithi089.stdout: "ceph version 19.3.0-5103-gf36283e7 (f36283e7c91cac1c01cb9a7c4ff21bd697dbd4f2) squid (dev)": 2 2024-09-21T04:14:15.966 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2024-09-21T04:14:15.966 INFO:teuthology.orchestra.run.smithi089.stdout: "osd": { 2024-09-21T04:14:15.966 INFO:teuthology.orchestra.run.smithi089.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-09-21T04:14:15.966 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2024-09-21T04:14:15.967 INFO:teuthology.orchestra.run.smithi089.stdout: "mds": { 2024-09-21T04:14:15.967 INFO:teuthology.orchestra.run.smithi089.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-21T04:14:15.967 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2024-09-21T04:14:15.967 INFO:teuthology.orchestra.run.smithi089.stdout: "overall": { 2024-09-21T04:14:15.967 INFO:teuthology.orchestra.run.smithi089.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 11, 2024-09-21T04:14:15.967 INFO:teuthology.orchestra.run.smithi089.stdout: "ceph version 19.3.0-5103-gf36283e7 (f36283e7c91cac1c01cb9a7c4ff21bd697dbd4f2) squid (dev)": 3 2024-09-21T04:14:15.968 INFO:teuthology.orchestra.run.smithi089.stdout: } 2024-09-21T04:14:15.968 INFO:teuthology.orchestra.run.smithi089.stdout:} 2024-09-21T04:14:16.338 INFO:teuthology.orchestra.run.smithi089.stdout: 5 131762 10.35 MB/sec execute 175 sec latency 74.604 ms 2024-09-21T04:14:16.412 INFO:teuthology.orchestra.run.smithi089.stdout:{ 2024-09-21T04:14:16.412 INFO:teuthology.orchestra.run.smithi089.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:2195e5f178be0a9390728e2a14088236d8fe6f82635f5c84418fe37a04457e00", 2024-09-21T04:14:16.412 INFO:teuthology.orchestra.run.smithi089.stdout: "in_progress": true, 2024-09-21T04:14:16.412 INFO:teuthology.orchestra.run.smithi089.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-21T04:14:16.412 INFO:teuthology.orchestra.run.smithi089.stdout: "services_complete": [ 2024-09-21T04:14:16.413 INFO:teuthology.orchestra.run.smithi089.stdout: "mgr" 2024-09-21T04:14:16.413 INFO:teuthology.orchestra.run.smithi089.stdout: ], 2024-09-21T04:14:16.413 INFO:teuthology.orchestra.run.smithi089.stdout: "progress": "3/23 daemons upgraded", 2024-09-21T04:14:16.413 INFO:teuthology.orchestra.run.smithi089.stdout: "message": "Currently upgrading mon daemons", 2024-09-21T04:14:16.413 INFO:teuthology.orchestra.run.smithi089.stdout: "is_paused": false 2024-09-21T04:14:16.413 INFO:teuthology.orchestra.run.smithi089.stdout:} 2024-09-21T04:14:16.875 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:14:16 smithi089 ceph-mon[64076]: from='client.34140 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T04:14:16.875 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:14:16 smithi089 ceph-mon[64076]: Upgrade: Updating mon.smithi195 2024-09-21T04:14:16.875 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:14:16 smithi089 ceph-mon[64076]: Deploying daemon mon.smithi195 on smithi195 2024-09-21T04:14:16.875 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:14:16 smithi089 ceph-mon[64076]: pgmap v10: 97 pgs: 97 active+clean; 330 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 21 KiB/s rd, 4.0 MiB/s wr, 413 op/s 2024-09-21T04:14:16.875 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:14:16 smithi089 ceph-mon[64076]: from='client.? 172.21.15.89:0/4211369355' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T04:14:16.995 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:16 smithi195 ceph-mon[25840]: from='client.34140 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T04:14:16.996 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:16 smithi195 ceph-mon[25840]: Upgrade: Updating mon.smithi195 2024-09-21T04:14:16.996 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:16 smithi195 ceph-mon[25840]: Deploying daemon mon.smithi195 on smithi195 2024-09-21T04:14:16.996 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:16 smithi195 ceph-mon[25840]: pgmap v10: 97 pgs: 97 active+clean; 330 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 21 KiB/s rd, 4.0 MiB/s wr, 413 op/s 2024-09-21T04:14:16.996 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:16 smithi195 ceph-mon[25840]: from='client.? 172.21.15.89:0/4211369355' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T04:14:17.032 INFO:teuthology.orchestra.run.smithi089.stdout:HEALTH_OK 2024-09-21T04:14:17.338 INFO:teuthology.orchestra.run.smithi089.stdout: 5 132051 10.31 MB/sec execute 176 sec latency 57.377 ms 2024-09-21T04:14:17.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:17 smithi195 systemd[1]: Stopping Ceph mon.smithi195 for 1b9b79fc-77ce-11ef-baf5-efdc52797490... 2024-09-21T04:14:18.109 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:17 smithi195 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-mon-smithi195[25824]: 2024-09-21T04:14:17.606+0000 7f720970d700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.smithi195 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-09-21T04:14:18.109 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:17 smithi195 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-mon-smithi195[25824]: 2024-09-21T04:14:17.606+0000 7f720970d700 -1 mon.smithi195@1(peon) e2 *** Got Signal Terminated *** 2024-09-21T04:14:18.109 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:17 smithi195 podman[49286]: 2024-09-21 04:14:17.822147321 +0000 UTC m=+0.354368759 container died 7f86c5b5d21ec6bddd0efbc610af337081e84b52867eb5264f18fb62268070d7 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-mon-smithi195, GIT_REPO=https://github.com/ceph/ceph-container.git, distribution-scope=public, vendor=Red Hat, Inc., name=centos-stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_CLEAN=True, maintainer=Guillaume Abrioux , io.openshift.tags=base centos centos-stream, GIT_BRANCH=HEAD, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vcs-type=git, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, com.redhat.component=centos-stream-container, io.buildah.version=1.19.8, RELEASE=HEAD, io.k8s.display-name=CentOS Stream 8, CEPH_POINT_RELEASE=-17.2.0, ceph=True, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, build-date=2022-05-03T08:36:31.336870, architecture=x86_64, version=8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, release=754, io.openshift.expose-services=) 2024-09-21T04:14:18.338 INFO:teuthology.orchestra.run.smithi089.stdout: 5 132254 10.26 MB/sec execute 177 sec latency 56.364 ms 2024-09-21T04:14:18.463 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:18 smithi195 podman[49286]: 2024-09-21 04:14:18.199733174 +0000 UTC m=+0.731954619 container cleanup 7f86c5b5d21ec6bddd0efbc610af337081e84b52867eb5264f18fb62268070d7 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-mon-smithi195, io.openshift.expose-services=, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_BRANCH=HEAD, architecture=x86_64, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_CLEAN=True, ceph=True, version=8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, release=754, vendor=Red Hat, Inc., build-date=2022-05-03T08:36:31.336870, distribution-scope=public, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.buildah.version=1.19.8, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=-17.2.0, io.k8s.display-name=CentOS Stream 8, name=centos-stream, vcs-type=git, com.redhat.component=centos-stream-container, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.openshift.tags=base centos centos-stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, RELEASE=HEAD) 2024-09-21T04:14:18.463 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:18 smithi195 bash[49286]: ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-mon-smithi195 2024-09-21T04:14:18.859 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:18 smithi195 podman[49296]: 2024-09-21 04:14:18.460265253 +0000 UTC m=+0.635261217 container remove 7f86c5b5d21ec6bddd0efbc610af337081e84b52867eb5264f18fb62268070d7 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-mon-smithi195, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_REPO=https://github.com/ceph/ceph-container.git, architecture=x86_64, build-date=2022-05-03T08:36:31.336870, version=8, io.k8s.display-name=CentOS Stream 8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.openshift.tags=base centos centos-stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.component=centos-stream-container, io.buildah.version=1.19.8, io.openshift.expose-services=, ceph=True, vendor=Red Hat, Inc., CEPH_POINT_RELEASE=-17.2.0, GIT_BRANCH=HEAD, release=754, name=centos-stream, distribution-scope=public, maintainer=Guillaume Abrioux , io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_CLEAN=True, RELEASE=HEAD, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vcs-type=git) 2024-09-21T04:14:18.859 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:18 smithi195 systemd[1]: ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490@mon.smithi195.service: Deactivated successfully. 2024-09-21T04:14:18.859 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:18 smithi195 systemd[1]: Stopped Ceph mon.smithi195 for 1b9b79fc-77ce-11ef-baf5-efdc52797490. 2024-09-21T04:14:18.859 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:18 smithi195 systemd[1]: ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490@mon.smithi195.service: Consumed 6.920s CPU time. 2024-09-21T04:14:19.338 INFO:teuthology.orchestra.run.smithi089.stdout: 5 132468 10.25 MB/sec execute 178 sec latency 104.762 ms 2024-09-21T04:14:19.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:18 smithi195 systemd[1]: Starting Ceph mon.smithi195 for 1b9b79fc-77ce-11ef-baf5-efdc52797490... 2024-09-21T04:14:19.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:19 smithi195 podman[49418]: 2024-09-21 04:14:19.062491532 +0000 UTC m=+0.020629979 image pull 48ed0a2cf89d6d0371a2c6185a5869a747265f6abd5ac2a74f00cc32d5628721 quay.ceph.io/ceph-ci/ceph@sha256:2195e5f178be0a9390728e2a14088236d8fe6f82635f5c84418fe37a04457e00 2024-09-21T04:14:19.645 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:19 smithi195 podman[49418]: 2024-09-21 04:14:19.377873766 +0000 UTC m=+0.336012202 container create 4255937444e7daceb33d28074f518a4cf14f676d8f19b06c8d560d11769055c3 (image=quay.ceph.io/ceph-ci/ceph@sha256:2195e5f178be0a9390728e2a14088236d8fe6f82635f5c84418fe37a04457e00, name=ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-mon-smithi195, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, ceph=True, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, RELEASE=wip-guits-main-2024-09-20-2310-f36283e, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.vendor=CentOS) 2024-09-21T04:14:19.645 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:19 smithi195 podman[49418]: 2024-09-21 04:14:19.614512288 +0000 UTC m=+0.572650727 container init 4255937444e7daceb33d28074f518a4cf14f676d8f19b06c8d560d11769055c3 (image=quay.ceph.io/ceph-ci/ceph@sha256:2195e5f178be0a9390728e2a14088236d8fe6f82635f5c84418fe37a04457e00, name=ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-mon-smithi195, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, GIT_CLEAN=True, GIT_BRANCH=HEAD, ceph=True, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-guits-main-2024-09-20-2310-f36283e, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.schema-version=1.0) 2024-09-21T04:14:19.645 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:19 smithi195 podman[49418]: 2024-09-21 04:14:19.618744567 +0000 UTC m=+0.576883007 container start 4255937444e7daceb33d28074f518a4cf14f676d8f19b06c8d560d11769055c3 (image=quay.ceph.io/ceph-ci/ceph@sha256:2195e5f178be0a9390728e2a14088236d8fe6f82635f5c84418fe37a04457e00, name=ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-mon-smithi195, GIT_BRANCH=HEAD, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, RELEASE=wip-guits-main-2024-09-20-2310-f36283e, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, GIT_CLEAN=True) 2024-09-21T04:14:20.110 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:19 smithi195 ceph-mon[49435]: set uid:gid to 167:167 (ceph:ceph) 2024-09-21T04:14:20.110 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:19 smithi195 ceph-mon[49435]: ceph version 19.3.0-5103-gf36283e7 (f36283e7c91cac1c01cb9a7c4ff21bd697dbd4f2) squid (dev), process ceph-mon, pid 2 2024-09-21T04:14:20.110 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:19 smithi195 ceph-mon[49435]: pidfile_write: ignore empty --pid-file 2024-09-21T04:14:20.110 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:19 smithi195 ceph-mon[49435]: load: jerasure load: lrc 2024-09-21T04:14:20.110 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:19 smithi195 ceph-mon[49435]: rocksdb: RocksDB version: 7.9.2 2024-09-21T04:14:20.110 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:19 smithi195 ceph-mon[49435]: rocksdb: Git sha 0 2024-09-21T04:14:20.110 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:19 smithi195 ceph-mon[49435]: rocksdb: Compile date 2024-09-20 23:32:13 2024-09-21T04:14:20.111 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:19 smithi195 ceph-mon[49435]: rocksdb: DB SUMMARY 2024-09-21T04:14:20.111 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:19 smithi195 ceph-mon[49435]: rocksdb: DB Session ID: K6HZM99RCARX4VNTNGC1 2024-09-21T04:14:20.111 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:19 smithi195 ceph-mon[49435]: rocksdb: CURRENT file: CURRENT 2024-09-21T04:14:20.111 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:19 smithi195 ceph-mon[49435]: rocksdb: IDENTITY file: IDENTITY 2024-09-21T04:14:20.111 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:19 smithi195 ceph-mon[49435]: rocksdb: MANIFEST file: MANIFEST-000009 size: 904 Bytes 2024-09-21T04:14:20.111 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:19 smithi195 ceph-mon[49435]: rocksdb: SST files in /var/lib/ceph/mon/ceph-smithi195/store.db dir, Total Num: 1, files: 000024.sst 2024-09-21T04:14:20.111 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:19 smithi195 ceph-mon[49435]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-smithi195/store.db: 000022.log size: 5981036 ; 2024-09-21T04:14:20.111 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:19 smithi195 ceph-mon[49435]: rocksdb: Options.error_if_exists: 0 2024-09-21T04:14:20.111 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:19 smithi195 ceph-mon[49435]: rocksdb: Options.create_if_missing: 0 2024-09-21T04:14:20.111 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:19 smithi195 ceph-mon[49435]: rocksdb: Options.paranoid_checks: 1 2024-09-21T04:14:20.112 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:19 smithi195 ceph-mon[49435]: rocksdb: Options.flush_verify_memtable_count: 1 2024-09-21T04:14:20.112 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:19 smithi195 ceph-mon[49435]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-09-21T04:14:20.112 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:19 smithi195 ceph-mon[49435]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-09-21T04:14:20.112 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:19 smithi195 ceph-mon[49435]: rocksdb: Options.env: 0x55b95529b120 2024-09-21T04:14:20.112 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:19 smithi195 ceph-mon[49435]: rocksdb: Options.fs: PosixFileSystem 2024-09-21T04:14:20.112 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:19 smithi195 ceph-mon[49435]: rocksdb: Options.info_log: 0x55b956b9f8e0 2024-09-21T04:14:20.113 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:19 smithi195 ceph-mon[49435]: rocksdb: Options.max_file_opening_threads: 16 2024-09-21T04:14:20.113 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:19 smithi195 ceph-mon[49435]: rocksdb: Options.statistics: (nil) 2024-09-21T04:14:20.113 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:19 smithi195 ceph-mon[49435]: rocksdb: Options.use_fsync: 0 2024-09-21T04:14:20.113 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:19 smithi195 ceph-mon[49435]: rocksdb: Options.max_log_file_size: 0 2024-09-21T04:14:20.113 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:19 smithi195 ceph-mon[49435]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-09-21T04:14:20.113 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:19 smithi195 ceph-mon[49435]: rocksdb: Options.log_file_time_to_roll: 0 2024-09-21T04:14:20.113 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:19 smithi195 ceph-mon[49435]: rocksdb: Options.keep_log_file_num: 1000 2024-09-21T04:14:20.114 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:19 smithi195 ceph-mon[49435]: rocksdb: Options.recycle_log_file_num: 0 2024-09-21T04:14:20.114 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:19 smithi195 ceph-mon[49435]: rocksdb: Options.allow_fallocate: 1 2024-09-21T04:14:20.114 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:19 smithi195 ceph-mon[49435]: rocksdb: Options.allow_mmap_reads: 0 2024-09-21T04:14:20.114 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:19 smithi195 ceph-mon[49435]: rocksdb: Options.allow_mmap_writes: 0 2024-09-21T04:14:20.114 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:19 smithi195 ceph-mon[49435]: rocksdb: Options.use_direct_reads: 0 2024-09-21T04:14:20.114 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:19 smithi195 ceph-mon[49435]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-09-21T04:14:20.114 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:19 smithi195 ceph-mon[49435]: rocksdb: Options.create_missing_column_families: 0 2024-09-21T04:14:20.114 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:19 smithi195 ceph-mon[49435]: rocksdb: Options.db_log_dir: 2024-09-21T04:14:20.114 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:19 smithi195 ceph-mon[49435]: rocksdb: Options.wal_dir: 2024-09-21T04:14:20.114 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:19 smithi195 ceph-mon[49435]: rocksdb: Options.table_cache_numshardbits: 6 2024-09-21T04:14:20.114 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:19 smithi195 ceph-mon[49435]: rocksdb: Options.WAL_ttl_seconds: 0 2024-09-21T04:14:20.114 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:19 smithi195 ceph-mon[49435]: rocksdb: Options.WAL_size_limit_MB: 0 2024-09-21T04:14:20.115 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:19 smithi195 ceph-mon[49435]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-09-21T04:14:20.115 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:19 smithi195 ceph-mon[49435]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-09-21T04:14:20.115 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:19 smithi195 ceph-mon[49435]: rocksdb: Options.is_fd_close_on_exec: 1 2024-09-21T04:14:20.115 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:19 smithi195 ceph-mon[49435]: rocksdb: Options.advise_random_on_open: 1 2024-09-21T04:14:20.115 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:19 smithi195 ceph-mon[49435]: rocksdb: Options.db_write_buffer_size: 0 2024-09-21T04:14:20.115 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:19 smithi195 ceph-mon[49435]: rocksdb: Options.write_buffer_manager: 0x55b956ba3b80 2024-09-21T04:14:20.115 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:19 smithi195 ceph-mon[49435]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-09-21T04:14:20.115 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:19 smithi195 ceph-mon[49435]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-09-21T04:14:20.115 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:19 smithi195 ceph-mon[49435]: rocksdb: Options.use_adaptive_mutex: 0 2024-09-21T04:14:20.115 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:19 smithi195 ceph-mon[49435]: rocksdb: Options.rate_limiter: (nil) 2024-09-21T04:14:20.115 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:19 smithi195 ceph-mon[49435]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-09-21T04:14:20.115 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:19 smithi195 ceph-mon[49435]: rocksdb: Options.wal_recovery_mode: 2 2024-09-21T04:14:20.115 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:19 smithi195 ceph-mon[49435]: rocksdb: Options.enable_thread_tracking: 0 2024-09-21T04:14:20.116 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:19 smithi195 ceph-mon[49435]: rocksdb: Options.enable_pipelined_write: 0 2024-09-21T04:14:20.116 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:19 smithi195 ceph-mon[49435]: rocksdb: Options.unordered_write: 0 2024-09-21T04:14:20.116 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:19 smithi195 ceph-mon[49435]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-09-21T04:14:20.116 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:19 smithi195 ceph-mon[49435]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-09-21T04:14:20.116 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:19 smithi195 ceph-mon[49435]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-09-21T04:14:20.116 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:19 smithi195 ceph-mon[49435]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-09-21T04:14:20.116 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:19 smithi195 ceph-mon[49435]: rocksdb: Options.row_cache: None 2024-09-21T04:14:20.116 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:19 smithi195 ceph-mon[49435]: rocksdb: Options.wal_filter: None 2024-09-21T04:14:20.117 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:19 smithi195 ceph-mon[49435]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-09-21T04:14:20.117 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:19 smithi195 ceph-mon[49435]: rocksdb: Options.allow_ingest_behind: 0 2024-09-21T04:14:20.117 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:19 smithi195 ceph-mon[49435]: rocksdb: Options.two_write_queues: 0 2024-09-21T04:14:20.117 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:19 smithi195 ceph-mon[49435]: rocksdb: Options.manual_wal_flush: 0 2024-09-21T04:14:20.117 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:19 smithi195 ceph-mon[49435]: rocksdb: Options.wal_compression: 0 2024-09-21T04:14:20.117 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:19 smithi195 ceph-mon[49435]: rocksdb: Options.atomic_flush: 0 2024-09-21T04:14:20.117 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:19 smithi195 ceph-mon[49435]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-09-21T04:14:20.117 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:19 smithi195 ceph-mon[49435]: rocksdb: Options.persist_stats_to_disk: 0 2024-09-21T04:14:20.117 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:19 smithi195 ceph-mon[49435]: rocksdb: Options.write_dbid_to_manifest: 0 2024-09-21T04:14:20.117 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:19 smithi195 ceph-mon[49435]: rocksdb: Options.log_readahead_size: 0 2024-09-21T04:14:20.117 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:19 smithi195 ceph-mon[49435]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-09-21T04:14:20.117 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:19 smithi195 ceph-mon[49435]: rocksdb: Options.best_efforts_recovery: 0 2024-09-21T04:14:20.117 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:19 smithi195 ceph-mon[49435]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-09-21T04:14:20.118 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:19 smithi195 ceph-mon[49435]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-09-21T04:14:20.118 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:19 smithi195 ceph-mon[49435]: rocksdb: Options.allow_data_in_errors: 0 2024-09-21T04:14:20.118 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:19 smithi195 ceph-mon[49435]: rocksdb: Options.db_host_id: __hostname__ 2024-09-21T04:14:20.119 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:19 smithi195 ceph-mon[49435]: rocksdb: Options.enforce_single_del_contracts: true 2024-09-21T04:14:20.119 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:19 smithi195 ceph-mon[49435]: rocksdb: Options.max_background_jobs: 2 2024-09-21T04:14:20.120 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:19 smithi195 ceph-mon[49435]: rocksdb: Options.max_background_compactions: -1 2024-09-21T04:14:20.120 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:19 smithi195 ceph-mon[49435]: rocksdb: Options.max_subcompactions: 1 2024-09-21T04:14:20.120 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:19 smithi195 ceph-mon[49435]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-09-21T04:14:20.121 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:19 smithi195 ceph-mon[49435]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-09-21T04:14:20.121 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:19 smithi195 ceph-mon[49435]: rocksdb: Options.delayed_write_rate : 16777216 2024-09-21T04:14:20.121 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:19 smithi195 ceph-mon[49435]: rocksdb: Options.max_total_wal_size: 0 2024-09-21T04:14:20.121 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:19 smithi195 ceph-mon[49435]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-09-21T04:14:20.121 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:19 smithi195 ceph-mon[49435]: rocksdb: Options.stats_dump_period_sec: 600 2024-09-21T04:14:20.121 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:19 smithi195 ceph-mon[49435]: rocksdb: Options.stats_persist_period_sec: 600 2024-09-21T04:14:20.121 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:19 smithi195 ceph-mon[49435]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-09-21T04:14:20.121 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:19 smithi195 ceph-mon[49435]: rocksdb: Options.max_open_files: -1 2024-09-21T04:14:20.121 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:19 smithi195 ceph-mon[49435]: rocksdb: Options.bytes_per_sync: 0 2024-09-21T04:14:20.121 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:19 smithi195 ceph-mon[49435]: rocksdb: Options.wal_bytes_per_sync: 0 2024-09-21T04:14:20.121 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:19 smithi195 ceph-mon[49435]: rocksdb: Options.strict_bytes_per_sync: 0 2024-09-21T04:14:20.122 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:19 smithi195 ceph-mon[49435]: rocksdb: Options.compaction_readahead_size: 0 2024-09-21T04:14:20.122 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:19 smithi195 ceph-mon[49435]: rocksdb: Options.max_background_flushes: -1 2024-09-21T04:14:20.122 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:19 smithi195 ceph-mon[49435]: rocksdb: Compression algorithms supported: 2024-09-21T04:14:20.122 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:19 smithi195 ceph-mon[49435]: rocksdb: kZSTD supported: 0 2024-09-21T04:14:20.122 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:19 smithi195 ceph-mon[49435]: rocksdb: kXpressCompression supported: 0 2024-09-21T04:14:20.122 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:19 smithi195 ceph-mon[49435]: rocksdb: kBZip2Compression supported: 0 2024-09-21T04:14:20.122 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:19 smithi195 ceph-mon[49435]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-09-21T04:14:20.122 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:19 smithi195 ceph-mon[49435]: rocksdb: kLZ4Compression supported: 1 2024-09-21T04:14:20.123 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:19 smithi195 ceph-mon[49435]: rocksdb: kZlibCompression supported: 1 2024-09-21T04:14:20.123 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:19 smithi195 ceph-mon[49435]: rocksdb: kLZ4HCCompression supported: 1 2024-09-21T04:14:20.123 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:19 smithi195 ceph-mon[49435]: rocksdb: kSnappyCompression supported: 1 2024-09-21T04:14:20.123 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:19 smithi195 ceph-mon[49435]: rocksdb: Fast CRC32 supported: Supported on x86 2024-09-21T04:14:20.123 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:19 smithi195 ceph-mon[49435]: rocksdb: DMutex implementation: pthread_mutex_t 2024-09-21T04:14:20.123 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:19 smithi195 ceph-mon[49435]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-smithi195/store.db/MANIFEST-000009 2024-09-21T04:14:20.123 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:19 smithi195 ceph-mon[49435]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-09-21T04:14:20.123 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:19 smithi195 ceph-mon[49435]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-09-21T04:14:20.123 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:19 smithi195 ceph-mon[49435]: rocksdb: Options.merge_operator: 2024-09-21T04:14:20.124 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:19 smithi195 ceph-mon[49435]: rocksdb: Options.compaction_filter: None 2024-09-21T04:14:20.124 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:19 smithi195 ceph-mon[49435]: rocksdb: Options.compaction_filter_factory: None 2024-09-21T04:14:20.124 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:19 smithi195 ceph-mon[49435]: rocksdb: Options.sst_partitioner_factory: None 2024-09-21T04:14:20.124 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:19 smithi195 ceph-mon[49435]: rocksdb: Options.memtable_factory: SkipListFactory 2024-09-21T04:14:20.124 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:19 smithi195 ceph-mon[49435]: rocksdb: Options.table_factory: BlockBasedTable 2024-09-21T04:14:20.124 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:19 smithi195 ceph-mon[49435]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55b956b9f540) 2024-09-21T04:14:20.124 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout: cache_index_and_filter_blocks: 1 2024-09-21T04:14:20.124 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-09-21T04:14:20.124 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-09-21T04:14:20.125 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout: pin_top_level_index_and_filter: 1 2024-09-21T04:14:20.125 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout: index_type: 0 2024-09-21T04:14:20.125 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout: data_block_index_type: 0 2024-09-21T04:14:20.125 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout: index_shortening: 1 2024-09-21T04:14:20.125 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout: data_block_hash_table_util_ratio: 0.750000 2024-09-21T04:14:20.125 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout: checksum: 4 2024-09-21T04:14:20.125 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout: no_block_cache: 0 2024-09-21T04:14:20.125 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout: block_cache: 0x55b956b93610 2024-09-21T04:14:20.125 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout: block_cache_name: BinnedLRUCache 2024-09-21T04:14:20.125 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout: block_cache_options: 2024-09-21T04:14:20.125 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout: capacity : 536870912 2024-09-21T04:14:20.125 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout: num_shard_bits : 4 2024-09-21T04:14:20.126 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout: strict_capacity_limit : 0 2024-09-21T04:14:20.126 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout: high_pri_pool_ratio: 0.000 2024-09-21T04:14:20.126 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout: block_cache_compressed: (nil) 2024-09-21T04:14:20.126 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout: persistent_cache: (nil) 2024-09-21T04:14:20.126 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout: block_size: 4096 2024-09-21T04:14:20.126 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout: block_size_deviation: 10 2024-09-21T04:14:20.126 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout: block_restart_interval: 16 2024-09-21T04:14:20.126 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout: index_block_restart_interval: 1 2024-09-21T04:14:20.127 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout: metadata_block_size: 4096 2024-09-21T04:14:20.127 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout: partition_filters: 0 2024-09-21T04:14:20.127 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout: use_delta_encoding: 1 2024-09-21T04:14:20.127 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout: filter_policy: bloomfilter 2024-09-21T04:14:20.127 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout: whole_key_filtering: 1 2024-09-21T04:14:20.127 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout: verify_compression: 0 2024-09-21T04:14:20.127 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout: read_amp_bytes_per_bit: 0 2024-09-21T04:14:20.127 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout: format_version: 5 2024-09-21T04:14:20.128 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout: enable_index_compression: 1 2024-09-21T04:14:20.128 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout: block_align: 0 2024-09-21T04:14:20.128 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout: max_auto_readahead_size: 262144 2024-09-21T04:14:20.128 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout: prepopulate_block_cache: 0 2024-09-21T04:14:20.128 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout: initial_auto_readahead_size: 8192 2024-09-21T04:14:20.128 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout: num_file_reads_for_auto_readahead: 2 2024-09-21T04:14:20.128 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:19 smithi195 ceph-mon[49435]: rocksdb: Options.write_buffer_size: 33554432 2024-09-21T04:14:20.128 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:19 smithi195 ceph-mon[49435]: rocksdb: Options.max_write_buffer_number: 2 2024-09-21T04:14:20.129 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:19 smithi195 ceph-mon[49435]: rocksdb: Options.compression: NoCompression 2024-09-21T04:14:20.129 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:19 smithi195 ceph-mon[49435]: rocksdb: Options.bottommost_compression: Disabled 2024-09-21T04:14:20.129 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:19 smithi195 ceph-mon[49435]: rocksdb: Options.prefix_extractor: nullptr 2024-09-21T04:14:20.129 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:19 smithi195 ceph-mon[49435]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-09-21T04:14:20.129 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:19 smithi195 ceph-mon[49435]: rocksdb: Options.num_levels: 7 2024-09-21T04:14:20.129 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:19 smithi195 ceph-mon[49435]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-09-21T04:14:20.129 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:19 smithi195 ceph-mon[49435]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-09-21T04:14:20.129 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:19 smithi195 ceph-mon[49435]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-09-21T04:14:20.130 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:19 smithi195 ceph-mon[49435]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-09-21T04:14:20.130 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:19 smithi195 ceph-mon[49435]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-09-21T04:14:20.130 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:19 smithi195 ceph-mon[49435]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-09-21T04:14:20.130 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:19 smithi195 ceph-mon[49435]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-09-21T04:14:20.130 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:19 smithi195 ceph-mon[49435]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-09-21T04:14:20.130 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:19 smithi195 ceph-mon[49435]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-09-21T04:14:20.130 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:19 smithi195 ceph-mon[49435]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-09-21T04:14:20.130 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:19 smithi195 ceph-mon[49435]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-09-21T04:14:20.131 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:19 smithi195 ceph-mon[49435]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-09-21T04:14:20.131 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:19 smithi195 ceph-mon[49435]: rocksdb: Options.compression_opts.window_bits: -14 2024-09-21T04:14:20.131 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:19 smithi195 ceph-mon[49435]: rocksdb: Options.compression_opts.level: 32767 2024-09-21T04:14:20.131 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:19 smithi195 ceph-mon[49435]: rocksdb: Options.compression_opts.strategy: 0 2024-09-21T04:14:20.131 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:19 smithi195 ceph-mon[49435]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-09-21T04:14:20.131 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:19 smithi195 ceph-mon[49435]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-09-21T04:14:20.132 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:19 smithi195 ceph-mon[49435]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-09-21T04:14:20.132 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:19 smithi195 ceph-mon[49435]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-09-21T04:14:20.132 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:19 smithi195 ceph-mon[49435]: rocksdb: Options.compression_opts.enabled: false 2024-09-21T04:14:20.132 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:19 smithi195 ceph-mon[49435]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-09-21T04:14:20.132 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:19 smithi195 ceph-mon[49435]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-09-21T04:14:20.132 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:19 smithi195 ceph-mon[49435]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-09-21T04:14:20.132 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:19 smithi195 ceph-mon[49435]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-09-21T04:14:20.132 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:19 smithi195 ceph-mon[49435]: rocksdb: Options.target_file_size_base: 67108864 2024-09-21T04:14:20.132 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:19 smithi195 ceph-mon[49435]: rocksdb: Options.target_file_size_multiplier: 1 2024-09-21T04:14:20.132 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:19 smithi195 ceph-mon[49435]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-09-21T04:14:20.132 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:19 smithi195 ceph-mon[49435]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-09-21T04:14:20.133 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:19 smithi195 ceph-mon[49435]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-09-21T04:14:20.133 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:19 smithi195 ceph-mon[49435]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-09-21T04:14:20.133 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:19 smithi195 ceph-mon[49435]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-09-21T04:14:20.133 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:19 smithi195 ceph-mon[49435]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-09-21T04:14:20.133 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:19 smithi195 ceph-mon[49435]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-09-21T04:14:20.133 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:19 smithi195 ceph-mon[49435]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-09-21T04:14:20.133 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:19 smithi195 ceph-mon[49435]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-09-21T04:14:20.133 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:19 smithi195 ceph-mon[49435]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-09-21T04:14:20.133 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:19 smithi195 ceph-mon[49435]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-09-21T04:14:20.133 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:19 smithi195 ceph-mon[49435]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-09-21T04:14:20.133 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:19 smithi195 ceph-mon[49435]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-09-21T04:14:20.133 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:19 smithi195 ceph-mon[49435]: rocksdb: Options.arena_block_size: 1048576 2024-09-21T04:14:20.133 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:19 smithi195 ceph-mon[49435]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-09-21T04:14:20.134 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:19 smithi195 ceph-mon[49435]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-09-21T04:14:20.134 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:19 smithi195 ceph-mon[49435]: rocksdb: Options.disable_auto_compactions: 0 2024-09-21T04:14:20.134 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:19 smithi195 ceph-mon[49435]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-09-21T04:14:20.134 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:19 smithi195 ceph-mon[49435]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-09-21T04:14:20.134 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:19 smithi195 ceph-mon[49435]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-09-21T04:14:20.134 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:19 smithi195 ceph-mon[49435]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-09-21T04:14:20.134 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:19 smithi195 ceph-mon[49435]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-09-21T04:14:20.134 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:19 smithi195 ceph-mon[49435]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-09-21T04:14:20.134 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:19 smithi195 ceph-mon[49435]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-09-21T04:14:20.134 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:19 smithi195 ceph-mon[49435]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-09-21T04:14:20.134 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:19 smithi195 ceph-mon[49435]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-09-21T04:14:20.134 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:19 smithi195 ceph-mon[49435]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-09-21T04:14:20.135 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:19 smithi195 ceph-mon[49435]: rocksdb: Options.table_properties_collectors: CompactOnDeletionCollector (Sliding window size = 32768 Deletion trigger = 16384 Deletion ratio = 0); 2024-09-21T04:14:20.135 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:19 smithi195 ceph-mon[49435]: rocksdb: Options.inplace_update_support: 0 2024-09-21T04:14:20.135 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:19 smithi195 ceph-mon[49435]: rocksdb: Options.inplace_update_num_locks: 10000 2024-09-21T04:14:20.135 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:19 smithi195 ceph-mon[49435]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-09-21T04:14:20.135 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:19 smithi195 ceph-mon[49435]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-09-21T04:14:20.135 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:19 smithi195 ceph-mon[49435]: rocksdb: Options.memtable_huge_page_size: 0 2024-09-21T04:14:20.135 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:19 smithi195 ceph-mon[49435]: rocksdb: Options.bloom_locality: 0 2024-09-21T04:14:20.135 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:19 smithi195 ceph-mon[49435]: rocksdb: Options.max_successive_merges: 0 2024-09-21T04:14:20.135 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:19 smithi195 ceph-mon[49435]: rocksdb: Options.optimize_filters_for_hits: 0 2024-09-21T04:14:20.135 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:19 smithi195 ceph-mon[49435]: rocksdb: Options.paranoid_file_checks: 0 2024-09-21T04:14:20.135 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:19 smithi195 ceph-mon[49435]: rocksdb: Options.force_consistency_checks: 1 2024-09-21T04:14:20.135 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:19 smithi195 ceph-mon[49435]: rocksdb: Options.report_bg_io_stats: 0 2024-09-21T04:14:20.135 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:19 smithi195 ceph-mon[49435]: rocksdb: Options.ttl: 2592000 2024-09-21T04:14:20.136 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:19 smithi195 ceph-mon[49435]: rocksdb: Options.periodic_compaction_seconds: 0 2024-09-21T04:14:20.136 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:19 smithi195 ceph-mon[49435]: rocksdb: Options.preclude_last_level_data_seconds: 0 2024-09-21T04:14:20.136 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:19 smithi195 ceph-mon[49435]: rocksdb: Options.preserve_internal_time_seconds: 0 2024-09-21T04:14:20.136 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:19 smithi195 ceph-mon[49435]: rocksdb: Options.enable_blob_files: false 2024-09-21T04:14:20.136 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:19 smithi195 ceph-mon[49435]: rocksdb: Options.min_blob_size: 0 2024-09-21T04:14:20.136 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:19 smithi195 ceph-mon[49435]: rocksdb: Options.blob_file_size: 268435456 2024-09-21T04:14:20.136 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:19 smithi195 ceph-mon[49435]: rocksdb: Options.blob_compression_type: NoCompression 2024-09-21T04:14:20.136 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:19 smithi195 ceph-mon[49435]: rocksdb: Options.enable_blob_garbage_collection: false 2024-09-21T04:14:20.136 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:19 smithi195 ceph-mon[49435]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-09-21T04:14:20.136 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:19 smithi195 ceph-mon[49435]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-09-21T04:14:20.136 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:19 smithi195 ceph-mon[49435]: rocksdb: Options.blob_compaction_readahead_size: 0 2024-09-21T04:14:20.136 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:19 smithi195 ceph-mon[49435]: rocksdb: Options.blob_file_starting_level: 0 2024-09-21T04:14:20.136 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:19 smithi195 ceph-mon[49435]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-09-21T04:14:20.137 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:19 smithi195 ceph-mon[49435]: rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 24.sst 2024-09-21T04:14:20.137 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:19 smithi195 ceph-mon[49435]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-09-21T04:14:20.137 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:19 smithi195 ceph-mon[49435]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-smithi195/store.db/MANIFEST-000009 succeeded,manifest_file_number is 9, next_file_number is 26, last_sequence is 12360, log_number is 22,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-09-21T04:14:20.137 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:19 smithi195 ceph-mon[49435]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 22 2024-09-21T04:14:20.137 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:19 smithi195 ceph-mon[49435]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: 3a087ec8-f1f5-4499-b31e-c7a57764da5d 2024-09-21T04:14:20.137 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:19 smithi195 ceph-mon[49435]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726892059650019, "job": 1, "event": "recovery_started", "wal_files": [22]} 2024-09-21T04:14:20.137 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:19 smithi195 ceph-mon[49435]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #22 mode 2 2024-09-21T04:14:20.137 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:19 smithi195 ceph-mon[49435]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726892059665622, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 27, "file_size": 3684679, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 12365, "largest_seqno": 13767, "table_properties": {"data_size": 3678137, "index_size": 4025, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 1477, "raw_key_size": 13275, "raw_average_key_size": 23, "raw_value_size": 3665669, "raw_average_value_size": 6453, "num_data_blocks": 183, "num_entries": 568, "num_filter_entries": 568, "num_deletions": 2, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[CompactOnDeletionCollector]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1726892059, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "3a087ec8-f1f5-4499-b31e-c7a57764da5d", "db_session_id": "K6HZM99RCARX4VNTNGC1", "orig_file_number": 27, "seqno_to_time_mapping": "N/A"}} 2024-09-21T04:14:20.137 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:19 smithi195 ceph-mon[49435]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726892059665798, "job": 1, "event": "recovery_finished"} 2024-09-21T04:14:20.137 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:19 smithi195 ceph-mon[49435]: rocksdb: [db/version_set.cc:5047] Creating manifest 29 2024-09-21T04:14:20.137 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:19 smithi195 ceph-mon[49435]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-09-21T04:14:20.137 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:19 smithi195 ceph-mon[49435]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-smithi195/store.db/000022.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-09-21T04:14:20.138 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:19 smithi195 ceph-mon[49435]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x55b956bc2e00 2024-09-21T04:14:20.138 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:19 smithi195 ceph-mon[49435]: rocksdb: DB pointer 0x55b956cc4000 2024-09-21T04:14:20.138 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:19 smithi195 ceph-mon[49435]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-09-21T04:14:20.138 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:19 smithi195 ceph-mon[49435]: rocksdb: [db/db_impl/db_impl.cc:1111] 2024-09-21T04:14:20.138 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout: ** DB Stats ** 2024-09-21T04:14:20.138 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-21T04:14:20.138 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-09-21T04:14:20.138 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-21T04:14:20.139 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-21T04:14:20.139 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-09-21T04:14:20.139 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-21T04:14:20.139 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-21T04:14:20.139 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout: 2024-09-21T04:14:20.140 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout: ** Compaction Stats [default] ** 2024-09-21T04:14:20.140 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-09-21T04:14:20.140 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-09-21T04:14:20.140 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout: L0 1/0 3.51 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 488.9 0.01 0.00 1 0.007 0 0 0.0 0.0 2024-09-21T04:14:20.140 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout: L6 1/0 8.01 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2024-09-21T04:14:20.140 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout: Sum 2/0 11.52 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 488.9 0.01 0.00 1 0.007 0 0 0.0 0.0 2024-09-21T04:14:20.140 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 488.9 0.01 0.00 1 0.007 0 0 0.0 0.0 2024-09-21T04:14:20.141 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout: 2024-09-21T04:14:20.141 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout: ** Compaction Stats [default] ** 2024-09-21T04:14:20.141 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-09-21T04:14:20.141 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-21T04:14:20.141 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 488.9 0.01 0.00 1 0.007 0 0 0.0 0.0 2024-09-21T04:14:20.141 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout: 2024-09-21T04:14:20.141 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-09-21T04:14:20.141 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout: 2024-09-21T04:14:20.142 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-21T04:14:20.142 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout: Flush(GB): cumulative 0.003, interval 0.003 2024-09-21T04:14:20.142 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-09-21T04:14:20.142 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-09-21T04:14:20.142 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-09-21T04:14:20.142 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout: AddFile(Keys): cumulative 0, interval 0 2024-09-21T04:14:20.142 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout: Cumulative compaction: 0.00 GB write, 189.15 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-21T04:14:20.142 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout: Interval compaction: 0.00 GB write, 189.15 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-21T04:14:20.142 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-09-21T04:14:20.142 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout: Block cache BinnedLRUCache@0x55b956b93610#2 capacity: 512.00 MB usage: 5.61 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1.1e-05 secs_since: 0 2024-09-21T04:14:20.142 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,1.53 KB,0.000292063%) IndexBlock(1,4.08 KB,0.000777841%) Misc(1,0.00 KB,0%) 2024-09-21T04:14:20.142 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout: 2024-09-21T04:14:20.143 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout: ** File Read Latency Histogram By Level [default] ** 2024-09-21T04:14:20.143 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:19 smithi195 ceph-mon[49435]: starting mon.smithi195 rank 1 at public addrs [v2:172.21.15.195:3300/0,v1:172.21.15.195:6789/0] at bind addrs [v2:172.21.15.195:3300/0,v1:172.21.15.195:6789/0] mon_data /var/lib/ceph/mon/ceph-smithi195 fsid 1b9b79fc-77ce-11ef-baf5-efdc52797490 2024-09-21T04:14:20.143 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:19 smithi195 ceph-mon[49435]: mon.smithi195@-1(???) e2 preinit fsid 1b9b79fc-77ce-11ef-baf5-efdc52797490 2024-09-21T04:14:20.143 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:19 smithi195 ceph-mon[49435]: mon.smithi195@-1(???).mds e6 new map 2024-09-21T04:14:20.143 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:19 smithi195 ceph-mon[49435]: mon.smithi195@-1(???).mds e6 print_map 2024-09-21T04:14:20.143 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout: e6 2024-09-21T04:14:20.143 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout: btime 1970-01-01T00:00:00:000000+0000 2024-09-21T04:14:20.143 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-09-21T04:14:20.143 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-09-21T04:14:20.144 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout: legacy client fscid: 1 2024-09-21T04:14:20.144 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout: 2024-09-21T04:14:20.144 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout: Filesystem 'foofs' (1) 2024-09-21T04:14:20.144 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout: fs_name foofs 2024-09-21T04:14:20.144 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout: epoch 4 2024-09-21T04:14:20.144 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout: flags 12 joinable allow_snaps allow_multimds_snaps 2024-09-21T04:14:20.144 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout: created 2024-09-21T04:08:42.283234+0000 2024-09-21T04:14:20.144 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout: modified 2024-09-21T04:08:48.134822+0000 2024-09-21T04:14:20.144 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout: tableserver 0 2024-09-21T04:14:20.144 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout: root 0 2024-09-21T04:14:20.144 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout: session_timeout 60 2024-09-21T04:14:20.144 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout: session_autoclose 300 2024-09-21T04:14:20.145 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout: max_file_size 1099511627776 2024-09-21T04:14:20.145 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout: max_xattr_size 65536 2024-09-21T04:14:20.145 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout: required_client_features {} 2024-09-21T04:14:20.145 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout: last_failure 0 2024-09-21T04:14:20.145 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout: last_failure_osd_epoch 0 2024-09-21T04:14:20.145 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-09-21T04:14:20.145 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout: max_mds 1 2024-09-21T04:14:20.145 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout: in 0 2024-09-21T04:14:20.145 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout: up {0=14500} 2024-09-21T04:14:20.145 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout: failed 2024-09-21T04:14:20.146 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout: damaged 2024-09-21T04:14:20.146 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout: stopped 2024-09-21T04:14:20.146 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout: data_pools [3] 2024-09-21T04:14:20.146 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout: metadata_pool 2 2024-09-21T04:14:20.146 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout: inline_data disabled 2024-09-21T04:14:20.146 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout: balancer 2024-09-21T04:14:20.146 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout: bal_rank_mask -1 2024-09-21T04:14:20.147 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout: standby_count_wanted 1 2024-09-21T04:14:20.147 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout: qdb_cluster leader: 0 members: 2024-09-21T04:14:20.147 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout: [mds.foofs.smithi089.osufws{0:14500} state up:active seq 2 join_fscid=1 addr [v2:172.21.15.89:6834/345305443,v1:172.21.15.89:6835/345305443] compat {c=[1],r=[1],i=[7ff]}] 2024-09-21T04:14:20.147 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout: 2024-09-21T04:14:20.147 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout: 2024-09-21T04:14:20.147 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout: Standby daemons: 2024-09-21T04:14:20.147 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout: 2024-09-21T04:14:20.147 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout: [mds.foofs.smithi195.hijxre{-1:24281} state up:standby seq 1 join_fscid=1 addr [v2:172.21.15.195:6832/3071482597,v1:172.21.15.195:6833/3071482597] compat {c=[1],r=[1],i=[7ff]}] 2024-09-21T04:14:20.147 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:19 smithi195 ceph-mon[49435]: mon.smithi195@-1(???).osd e74 crush map has features 3314933000854323200, adjusting msgr requires 2024-09-21T04:14:20.147 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:19 smithi195 ceph-mon[49435]: mon.smithi195@-1(???).osd e74 crush map has features 432629239337189376, adjusting msgr requires 2024-09-21T04:14:20.148 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:19 smithi195 ceph-mon[49435]: mon.smithi195@-1(???).osd e74 crush map has features 432629239337189376, adjusting msgr requires 2024-09-21T04:14:20.148 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:19 smithi195 ceph-mon[49435]: mon.smithi195@-1(???).osd e74 crush map has features 432629239337189376, adjusting msgr requires 2024-09-21T04:14:20.148 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:19 smithi195 ceph-mon[49435]: mon.smithi195@-1(???).paxosservice(auth 1..30) refresh upgraded, format 0 -> 3 2024-09-21T04:14:20.148 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:19 smithi195 bash[49418]: 4255937444e7daceb33d28074f518a4cf14f676d8f19b06c8d560d11769055c3 2024-09-21T04:14:20.148 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:19 smithi195 systemd[1]: Started Ceph mon.smithi195 for 1b9b79fc-77ce-11ef-baf5-efdc52797490. 2024-09-21T04:14:20.338 INFO:teuthology.orchestra.run.smithi089.stdout: 5 132769 10.23 MB/sec execute 179 sec latency 56.668 ms 2024-09-21T04:14:20.953 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:14:20 smithi089 ceph-mon[64076]: from='client.34148 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T04:14:20.953 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:14:20 smithi089 ceph-mon[64076]: from='client.? 172.21.15.89:0/142114250' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-21T04:14:20.953 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:14:20 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T04:14:21.109 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:20 smithi195 ceph-mon[49435]: from='client.34148 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T04:14:21.109 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:20 smithi195 ceph-mon[49435]: from='client.? 172.21.15.89:0/142114250' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-21T04:14:21.109 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:20 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T04:14:21.338 INFO:teuthology.orchestra.run.smithi089.stdout: 5 133050 10.22 MB/sec execute 180 sec latency 109.453 ms 2024-09-21T04:14:22.008 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:14:21 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "mon metadata", "id": "smithi089"} : dispatch 2024-09-21T04:14:22.008 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:14:21 smithi089 ceph-mon[64076]: mon.smithi089 calling monitor election 2024-09-21T04:14:22.009 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:14:21 smithi089 ceph-mon[64076]: mon.smithi195 calling monitor election 2024-09-21T04:14:22.009 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:14:21 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "mon metadata", "id": "smithi195"} : dispatch 2024-09-21T04:14:22.009 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:14:21 smithi089 ceph-mon[64076]: mon.smithi089 is new leader, mons smithi089,smithi195 in quorum (ranks 0,1) 2024-09-21T04:14:22.009 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:14:21 smithi089 ceph-mon[64076]: monmap epoch 3 2024-09-21T04:14:22.009 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:14:21 smithi089 ceph-mon[64076]: fsid 1b9b79fc-77ce-11ef-baf5-efdc52797490 2024-09-21T04:14:22.009 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:14:21 smithi089 ceph-mon[64076]: last_changed 2024-09-21T04:14:20.848291+0000 2024-09-21T04:14:22.009 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:14:21 smithi089 ceph-mon[64076]: created 2024-09-21T04:01:28.028960+0000 2024-09-21T04:14:22.009 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:14:21 smithi089 ceph-mon[64076]: min_mon_release 19 (squid) 2024-09-21T04:14:22.009 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:14:21 smithi089 ceph-mon[64076]: election_strategy: 1 2024-09-21T04:14:22.009 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:14:21 smithi089 ceph-mon[64076]: 0: [v2:172.21.15.89:3300/0,v1:172.21.15.89:6789/0] mon.smithi089 2024-09-21T04:14:22.010 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:14:21 smithi089 ceph-mon[64076]: 1: [v2:172.21.15.195:3300/0,v1:172.21.15.195:6789/0] mon.smithi195 2024-09-21T04:14:22.010 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:14:21 smithi089 ceph-mon[64076]: fsmap foofs:1 {0=foofs.smithi089.osufws=up:active} 1 up:standby 2024-09-21T04:14:22.010 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:14:21 smithi089 ceph-mon[64076]: osdmap e74: 8 total, 8 up, 8 in 2024-09-21T04:14:22.010 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:14:21 smithi089 ceph-mon[64076]: mgrmap e43: smithi089.srlyxf(active, since 18s), standbys: smithi195.cgotzm 2024-09-21T04:14:22.010 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:14:21 smithi089 ceph-mon[64076]: overall HEALTH_OK 2024-09-21T04:14:22.010 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:14:21 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:14:22.010 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:14:21 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:14:22.010 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:14:21 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T04:14:22.273 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:21 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "mon metadata", "id": "smithi089"} : dispatch 2024-09-21T04:14:22.273 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:21 smithi195 ceph-mon[49435]: mon.smithi089 calling monitor election 2024-09-21T04:14:22.273 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:21 smithi195 ceph-mon[49435]: mon.smithi195 calling monitor election 2024-09-21T04:14:22.273 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:21 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "mon metadata", "id": "smithi195"} : dispatch 2024-09-21T04:14:22.273 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:21 smithi195 ceph-mon[49435]: mon.smithi089 is new leader, mons smithi089,smithi195 in quorum (ranks 0,1) 2024-09-21T04:14:22.274 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:21 smithi195 ceph-mon[49435]: monmap epoch 3 2024-09-21T04:14:22.274 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:21 smithi195 ceph-mon[49435]: fsid 1b9b79fc-77ce-11ef-baf5-efdc52797490 2024-09-21T04:14:22.274 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:21 smithi195 ceph-mon[49435]: last_changed 2024-09-21T04:14:20.848291+0000 2024-09-21T04:14:22.274 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:21 smithi195 ceph-mon[49435]: created 2024-09-21T04:01:28.028960+0000 2024-09-21T04:14:22.274 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:21 smithi195 ceph-mon[49435]: min_mon_release 19 (squid) 2024-09-21T04:14:22.274 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:21 smithi195 ceph-mon[49435]: election_strategy: 1 2024-09-21T04:14:22.274 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:21 smithi195 ceph-mon[49435]: 0: [v2:172.21.15.89:3300/0,v1:172.21.15.89:6789/0] mon.smithi089 2024-09-21T04:14:22.274 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:21 smithi195 ceph-mon[49435]: 1: [v2:172.21.15.195:3300/0,v1:172.21.15.195:6789/0] mon.smithi195 2024-09-21T04:14:22.274 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:21 smithi195 ceph-mon[49435]: fsmap foofs:1 {0=foofs.smithi089.osufws=up:active} 1 up:standby 2024-09-21T04:14:22.274 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:21 smithi195 ceph-mon[49435]: osdmap e74: 8 total, 8 up, 8 in 2024-09-21T04:14:22.274 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:21 smithi195 ceph-mon[49435]: mgrmap e43: smithi089.srlyxf(active, since 18s), standbys: smithi195.cgotzm 2024-09-21T04:14:22.274 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:21 smithi195 ceph-mon[49435]: overall HEALTH_OK 2024-09-21T04:14:22.274 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:21 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:14:22.274 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:21 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:14:22.275 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:21 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T04:14:22.338 INFO:teuthology.orchestra.run.smithi089.stdout: 5 133339 10.17 MB/sec execute 181 sec latency 83.004 ms 2024-09-21T04:14:23.025 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:14:22 smithi089 ceph-mon[64076]: pgmap v13: 97 pgs: 97 active+clean; 336 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 7.0 KiB/s rd, 4.3 MiB/s wr, 430 op/s 2024-09-21T04:14:23.178 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:22 smithi195 ceph-mon[49435]: pgmap v13: 97 pgs: 97 active+clean; 336 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 7.0 KiB/s rd, 4.3 MiB/s wr, 430 op/s 2024-09-21T04:14:23.338 INFO:teuthology.orchestra.run.smithi089.stdout: 5 133764 10.17 MB/sec execute 182 sec latency 55.391 ms 2024-09-21T04:14:24.338 INFO:teuthology.orchestra.run.smithi089.stdout: 5 134015 10.18 MB/sec execute 183 sec latency 50.900 ms 2024-09-21T04:14:25.182 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:14:24 smithi089 ceph-mon[64076]: pgmap v14: 97 pgs: 97 active+clean; 326 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 5.6 MiB/s wr, 609 op/s 2024-09-21T04:14:25.182 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:14:24 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:14:25.182 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:14:24 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:14:25.338 INFO:teuthology.orchestra.run.smithi089.stdout: 5 134486 10.23 MB/sec execute 184 sec latency 39.765 ms 2024-09-21T04:14:25.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:24 smithi195 ceph-mon[49435]: pgmap v14: 97 pgs: 97 active+clean; 326 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 5.6 MiB/s wr, 609 op/s 2024-09-21T04:14:25.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:24 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:14:25.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:24 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:14:26.195 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:14:26 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:14:26.195 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:14:26 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:14:26.195 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:14:26 smithi089 ceph-mon[64076]: pgmap v15: 97 pgs: 97 active+clean; 326 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 3.8 MiB/s wr, 394 op/s 2024-09-21T04:14:26.338 INFO:teuthology.orchestra.run.smithi089.stdout: 5 134900 10.26 MB/sec execute 185 sec latency 34.973 ms 2024-09-21T04:14:26.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:26 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:14:26.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:26 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:14:26.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:26 smithi195 ceph-mon[49435]: pgmap v15: 97 pgs: 97 active+clean; 326 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 3.8 MiB/s wr, 394 op/s 2024-09-21T04:14:27.338 INFO:teuthology.orchestra.run.smithi089.stdout: 5 135145 10.25 MB/sec execute 186 sec latency 53.008 ms 2024-09-21T04:14:28.338 INFO:teuthology.orchestra.run.smithi089.stdout: 5 135423 10.21 MB/sec execute 187 sec latency 41.731 ms 2024-09-21T04:14:28.994 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:14:28 smithi089 ceph-mon[64076]: pgmap v16: 97 pgs: 97 active+clean; 326 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 3.8 MiB/s wr, 394 op/s 2024-09-21T04:14:28.994 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:14:28 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:14:28.994 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:14:28 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:14:28.994 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:14:28 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T04:14:28.994 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:14:28 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T04:14:28.995 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:14:28 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:14:28.995 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:14:28 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:14:28.995 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:14:28 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-09-21T04:14:28.995 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:14:28 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-09-21T04:14:28.995 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:14:28 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T04:14:29.339 INFO:teuthology.orchestra.run.smithi089.stdout: 5 135706 10.17 MB/sec execute 188 sec latency 55.282 ms 2024-09-21T04:14:29.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:28 smithi195 ceph-mon[49435]: pgmap v16: 97 pgs: 97 active+clean; 326 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 3.8 MiB/s wr, 394 op/s 2024-09-21T04:14:29.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:28 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:14:29.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:28 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:14:29.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:28 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T04:14:29.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:28 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T04:14:29.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:28 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:14:29.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:28 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:14:29.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:28 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-09-21T04:14:29.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:28 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-09-21T04:14:29.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:28 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T04:14:30.039 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:14:29 smithi089 ceph-mon[64076]: Reconfiguring mon.smithi089 (monmap changed)... 2024-09-21T04:14:30.039 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:14:29 smithi089 ceph-mon[64076]: Reconfiguring daemon mon.smithi089 on smithi089 2024-09-21T04:14:30.339 INFO:teuthology.orchestra.run.smithi089.stdout: 5 135993 10.15 MB/sec execute 189 sec latency 47.758 ms 2024-09-21T04:14:30.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:29 smithi195 ceph-mon[49435]: Reconfiguring mon.smithi089 (monmap changed)... 2024-09-21T04:14:30.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:29 smithi195 ceph-mon[49435]: Reconfiguring daemon mon.smithi089 on smithi089 2024-09-21T04:14:30.974 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:14:30 smithi089 ceph-mon[64076]: pgmap v17: 97 pgs: 97 active+clean; 336 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 8.3 KiB/s rd, 6.5 MiB/s wr, 594 op/s 2024-09-21T04:14:30.974 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:14:30 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:14:30.975 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:14:30 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:14:30.975 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:14:30 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "auth get-or-create", "entity": "mgr.smithi089.srlyxf", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]} : dispatch 2024-09-21T04:14:31.049 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:14:30 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "mgr services"} : dispatch 2024-09-21T04:14:31.049 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:14:30 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T04:14:31.339 INFO:teuthology.orchestra.run.smithi089.stdout: 5 136337 10.15 MB/sec execute 190 sec latency 44.196 ms 2024-09-21T04:14:31.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:30 smithi195 ceph-mon[49435]: pgmap v17: 97 pgs: 97 active+clean; 336 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 8.3 KiB/s rd, 6.5 MiB/s wr, 594 op/s 2024-09-21T04:14:31.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:30 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:14:31.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:30 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:14:31.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:30 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "auth get-or-create", "entity": "mgr.smithi089.srlyxf", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]} : dispatch 2024-09-21T04:14:31.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:30 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "mgr services"} : dispatch 2024-09-21T04:14:31.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:30 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T04:14:32.078 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:14:31 smithi089 ceph-mon[64076]: Reconfiguring mgr.smithi089.srlyxf (monmap changed)... 2024-09-21T04:14:32.079 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:14:31 smithi089 ceph-mon[64076]: Reconfiguring daemon mgr.smithi089.srlyxf on smithi089 2024-09-21T04:14:32.339 INFO:teuthology.orchestra.run.smithi089.stdout: 5 136793 10.14 MB/sec execute 191 sec latency 47.683 ms 2024-09-21T04:14:32.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:31 smithi195 ceph-mon[49435]: Reconfiguring mgr.smithi089.srlyxf (monmap changed)... 2024-09-21T04:14:32.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:31 smithi195 ceph-mon[49435]: Reconfiguring daemon mgr.smithi089.srlyxf on smithi089 2024-09-21T04:14:33.103 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:14:32 smithi089 ceph-mon[64076]: pgmap v18: 97 pgs: 97 active+clean; 336 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 5.1 KiB/s rd, 3.9 MiB/s wr, 377 op/s 2024-09-21T04:14:33.104 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:14:32 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T04:14:33.339 INFO:teuthology.orchestra.run.smithi089.stdout: 5 137286 10.13 MB/sec execute 192 sec latency 39.418 ms 2024-09-21T04:14:33.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:32 smithi195 ceph-mon[49435]: pgmap v18: 97 pgs: 97 active+clean; 336 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 5.1 KiB/s rd, 3.9 MiB/s wr, 377 op/s 2024-09-21T04:14:33.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:32 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T04:14:34.339 INFO:teuthology.orchestra.run.smithi089.stdout: 5 137711 10.17 MB/sec execute 193 sec latency 43.211 ms 2024-09-21T04:14:34.873 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:14:34 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:14:34.873 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:14:34 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:14:34.873 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:14:34 smithi089 ceph-mon[64076]: Reconfiguring crash.smithi089 (monmap changed)... 2024-09-21T04:14:34.873 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:14:34 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "auth get-or-create", "entity": "client.crash.smithi089", "caps": ["mon", "profile crash", "mgr", "profile crash"]} : dispatch 2024-09-21T04:14:34.873 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:14:34 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T04:14:34.873 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:14:34 smithi089 ceph-mon[64076]: Reconfiguring daemon crash.smithi089 on smithi089 2024-09-21T04:14:34.873 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:14:34 smithi089 ceph-mon[64076]: pgmap v19: 97 pgs: 97 active+clean; 326 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 9.2 KiB/s rd, 5.6 MiB/s wr, 603 op/s 2024-09-21T04:14:35.109 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:34 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:14:35.109 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:34 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:14:35.109 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:34 smithi195 ceph-mon[49435]: Reconfiguring crash.smithi089 (monmap changed)... 2024-09-21T04:14:35.109 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:34 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "auth get-or-create", "entity": "client.crash.smithi089", "caps": ["mon", "profile crash", "mgr", "profile crash"]} : dispatch 2024-09-21T04:14:35.109 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:34 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T04:14:35.109 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:34 smithi195 ceph-mon[49435]: Reconfiguring daemon crash.smithi089 on smithi089 2024-09-21T04:14:35.109 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:34 smithi195 ceph-mon[49435]: pgmap v19: 97 pgs: 97 active+clean; 326 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 9.2 KiB/s rd, 5.6 MiB/s wr, 603 op/s 2024-09-21T04:14:35.339 INFO:teuthology.orchestra.run.smithi089.stdout: 5 138354 10.26 MB/sec execute 194 sec latency 40.401 ms 2024-09-21T04:14:36.339 INFO:teuthology.orchestra.run.smithi089.stdout: 5 138696 10.27 MB/sec execute 195 sec latency 65.242 ms 2024-09-21T04:14:37.165 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:14:36 smithi089 ceph-mon[64076]: pgmap v20: 97 pgs: 97 active+clean; 326 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.3 MiB/s wr, 424 op/s 2024-09-21T04:14:37.165 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:14:36 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:14:37.165 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:14:36 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:14:37.165 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:14:36 smithi089 ceph-mon[64076]: Reconfiguring osd.0 (monmap changed)... 2024-09-21T04:14:37.165 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:14:36 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "auth get", "entity": "osd.0"} : dispatch 2024-09-21T04:14:37.165 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:14:36 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T04:14:37.165 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:14:36 smithi089 ceph-mon[64076]: Reconfiguring daemon osd.0 on smithi089 2024-09-21T04:14:37.339 INFO:teuthology.orchestra.run.smithi089.stdout: 5 139107 10.23 MB/sec execute 196 sec latency 34.809 ms 2024-09-21T04:14:37.360 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:36 smithi195 ceph-mon[49435]: pgmap v20: 97 pgs: 97 active+clean; 326 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.3 MiB/s wr, 424 op/s 2024-09-21T04:14:37.360 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:36 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:14:37.360 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:36 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:14:37.360 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:36 smithi195 ceph-mon[49435]: Reconfiguring osd.0 (monmap changed)... 2024-09-21T04:14:37.360 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:36 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "auth get", "entity": "osd.0"} : dispatch 2024-09-21T04:14:37.360 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:36 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T04:14:37.360 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:36 smithi195 ceph-mon[49435]: Reconfiguring daemon osd.0 on smithi089 2024-09-21T04:14:38.339 INFO:teuthology.orchestra.run.smithi089.stdout: 5 139430 10.22 MB/sec execute 197 sec latency 130.260 ms 2024-09-21T04:14:39.201 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:14:38 smithi089 ceph-mon[64076]: pgmap v21: 97 pgs: 97 active+clean; 326 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.3 MiB/s wr, 424 op/s 2024-09-21T04:14:39.201 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:14:38 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:14:39.201 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:14:38 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:14:39.201 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:14:38 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "auth get", "entity": "osd.1"} : dispatch 2024-09-21T04:14:39.201 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:14:38 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T04:14:39.339 INFO:teuthology.orchestra.run.smithi089.stdout: 5 139775 10.22 MB/sec execute 198 sec latency 48.413 ms 2024-09-21T04:14:39.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:38 smithi195 ceph-mon[49435]: pgmap v21: 97 pgs: 97 active+clean; 326 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.3 MiB/s wr, 424 op/s 2024-09-21T04:14:39.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:38 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:14:39.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:38 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:14:39.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:38 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "auth get", "entity": "osd.1"} : dispatch 2024-09-21T04:14:39.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:38 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T04:14:40.229 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:14:39 smithi089 ceph-mon[64076]: Reconfiguring osd.1 (monmap changed)... 2024-09-21T04:14:40.229 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:14:39 smithi089 ceph-mon[64076]: Reconfiguring daemon osd.1 on smithi089 2024-09-21T04:14:40.339 INFO:teuthology.orchestra.run.smithi089.stdout: 5 140223 10.21 MB/sec execute 199 sec latency 40.839 ms 2024-09-21T04:14:40.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:40 smithi195 ceph-mon[49435]: Reconfiguring osd.1 (monmap changed)... 2024-09-21T04:14:40.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:40 smithi195 ceph-mon[49435]: Reconfiguring daemon osd.1 on smithi089 2024-09-21T04:14:41.236 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:14:40 smithi089 ceph-mon[64076]: pgmap v22: 97 pgs: 97 active+clean; 341 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 8.9 KiB/s rd, 7.5 MiB/s wr, 688 op/s 2024-09-21T04:14:41.339 INFO:teuthology.orchestra.run.smithi089.stdout: 5 140602 10.18 MB/sec execute 200 sec latency 59.278 ms 2024-09-21T04:14:41.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:40 smithi195 ceph-mon[49435]: pgmap v22: 97 pgs: 97 active+clean; 341 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 8.9 KiB/s rd, 7.5 MiB/s wr, 688 op/s 2024-09-21T04:14:42.259 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:14:42 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:14:42.259 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:14:42 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:14:42.259 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:14:42 smithi089 ceph-mon[64076]: Reconfiguring osd.2 (monmap changed)... 2024-09-21T04:14:42.259 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:14:42 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "auth get", "entity": "osd.2"} : dispatch 2024-09-21T04:14:42.259 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:14:42 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T04:14:42.259 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:14:42 smithi089 ceph-mon[64076]: Reconfiguring daemon osd.2 on smithi089 2024-09-21T04:14:42.259 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:14:42 smithi089 ceph-mon[64076]: pgmap v23: 97 pgs: 97 active+clean; 341 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 6.8 KiB/s rd, 4.8 MiB/s wr, 488 op/s 2024-09-21T04:14:42.339 INFO:teuthology.orchestra.run.smithi089.stdout: 5 140898 10.17 MB/sec execute 201 sec latency 72.790 ms 2024-09-21T04:14:42.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:42 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:14:42.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:42 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:14:42.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:42 smithi195 ceph-mon[49435]: Reconfiguring osd.2 (monmap changed)... 2024-09-21T04:14:42.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:42 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "auth get", "entity": "osd.2"} : dispatch 2024-09-21T04:14:42.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:42 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T04:14:42.610 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:42 smithi195 ceph-mon[49435]: Reconfiguring daemon osd.2 on smithi089 2024-09-21T04:14:42.610 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:42 smithi195 ceph-mon[49435]: pgmap v23: 97 pgs: 97 active+clean; 341 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 6.8 KiB/s rd, 4.8 MiB/s wr, 488 op/s 2024-09-21T04:14:43.339 INFO:teuthology.orchestra.run.smithi089.stdout: 5 141685 10.28 MB/sec execute 202 sec latency 43.344 ms 2024-09-21T04:14:44.339 INFO:teuthology.orchestra.run.smithi089.stdout: 5 142318 10.34 MB/sec execute 203 sec latency 22.310 ms 2024-09-21T04:14:44.772 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:14:44 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:14:44.772 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:14:44 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:14:44.772 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:14:44 smithi089 ceph-mon[64076]: Reconfiguring osd.3 (monmap changed)... 2024-09-21T04:14:44.772 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:14:44 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "auth get", "entity": "osd.3"} : dispatch 2024-09-21T04:14:44.772 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:14:44 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T04:14:44.772 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:14:44 smithi089 ceph-mon[64076]: Reconfiguring daemon osd.3 on smithi089 2024-09-21T04:14:44.772 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:14:44 smithi089 ceph-mon[64076]: pgmap v24: 97 pgs: 97 active+clean; 332 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 7.1 MiB/s wr, 734 op/s 2024-09-21T04:14:44.859 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:44 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:14:44.859 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:44 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:14:44.859 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:44 smithi195 ceph-mon[49435]: Reconfiguring osd.3 (monmap changed)... 2024-09-21T04:14:44.859 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:44 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "auth get", "entity": "osd.3"} : dispatch 2024-09-21T04:14:44.859 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:44 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T04:14:44.859 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:44 smithi195 ceph-mon[49435]: Reconfiguring daemon osd.3 on smithi089 2024-09-21T04:14:44.859 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:44 smithi195 ceph-mon[49435]: pgmap v24: 97 pgs: 97 active+clean; 332 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 7.1 MiB/s wr, 734 op/s 2024-09-21T04:14:45.339 INFO:teuthology.orchestra.run.smithi089.stdout: 5 142796 10.31 MB/sec execute 204 sec latency 31.695 ms 2024-09-21T04:14:46.339 INFO:teuthology.orchestra.run.smithi089.stdout: 5 143217 10.31 MB/sec execute 205 sec latency 36.647 ms 2024-09-21T04:14:47.290 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:14:47 smithi089 ceph-mon[64076]: pgmap v25: 97 pgs: 97 active+clean; 332 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 7.0 KiB/s rd, 5.5 MiB/s wr, 508 op/s 2024-09-21T04:14:47.290 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:14:47 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:14:47.290 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:14:47 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:14:47.290 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:14:47 smithi089 ceph-mon[64076]: Reconfiguring mds.foofs.smithi089.osufws (monmap changed)... 2024-09-21T04:14:47.290 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:14:47 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "auth get-or-create", "entity": "mds.foofs.smithi089.osufws", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-09-21T04:14:47.290 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:14:47 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T04:14:47.290 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:14:47 smithi089 ceph-mon[64076]: Reconfiguring daemon mds.foofs.smithi089.osufws on smithi089 2024-09-21T04:14:47.339 INFO:teuthology.orchestra.run.smithi089.stdout: 5 143895 10.33 MB/sec execute 206 sec latency 37.140 ms 2024-09-21T04:14:47.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:47 smithi195 ceph-mon[49435]: pgmap v25: 97 pgs: 97 active+clean; 332 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 7.0 KiB/s rd, 5.5 MiB/s wr, 508 op/s 2024-09-21T04:14:47.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:47 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:14:47.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:47 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:14:47.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:47 smithi195 ceph-mon[49435]: Reconfiguring mds.foofs.smithi089.osufws (monmap changed)... 2024-09-21T04:14:47.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:47 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "auth get-or-create", "entity": "mds.foofs.smithi089.osufws", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-09-21T04:14:47.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:47 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T04:14:47.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:47 smithi195 ceph-mon[49435]: Reconfiguring daemon mds.foofs.smithi089.osufws on smithi089 2024-09-21T04:14:47.662 INFO:teuthology.orchestra.run.smithi089.stdout:true 2024-09-21T04:14:48.297 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:14:48 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T04:14:48.339 INFO:teuthology.orchestra.run.smithi089.stdout: 5 144431 10.34 MB/sec execute 207 sec latency 78.496 ms 2024-09-21T04:14:48.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:48 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T04:14:48.855 INFO:teuthology.orchestra.run.smithi089.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-21T04:14:48.855 INFO:teuthology.orchestra.run.smithi089.stdout:alertmanager.smithi089 smithi089 *:9093,9094 running (3m) 37s ago 11m 19.5M - 0.27.0 11f11916f8cd 65dfcbcafee1 2024-09-21T04:14:48.855 INFO:teuthology.orchestra.run.smithi089.stdout:crash.smithi089 smithi089 running (11m) 37s ago 11m 6409k - 17.2.0 e1d6a67b021e d1e0a8969f78 2024-09-21T04:14:48.855 INFO:teuthology.orchestra.run.smithi089.stdout:crash.smithi195 smithi195 running (11m) 24s ago 11m 6400k - 17.2.0 e1d6a67b021e 7e213ff76084 2024-09-21T04:14:48.855 INFO:teuthology.orchestra.run.smithi089.stdout:grafana.smithi089 smithi089 *:3000 running (2m) 37s ago 11m 45.0M - dad864ee21e9 ec4d6738bce7 2024-09-21T04:14:48.856 INFO:teuthology.orchestra.run.smithi089.stdout:mds.foofs.smithi089.osufws smithi089 running (6m) 37s ago 6m 37.9M - 17.2.0 e1d6a67b021e 664d41859bea 2024-09-21T04:14:48.856 INFO:teuthology.orchestra.run.smithi089.stdout:mds.foofs.smithi195.hijxre smithi195 running (6m) 24s ago 6m 16.6M - 17.2.0 e1d6a67b021e 7bc0c6542d12 2024-09-21T04:14:48.856 INFO:teuthology.orchestra.run.smithi089.stdout:mgr.smithi089.srlyxf smithi089 *:8443,9283,8765 running (2m) 37s ago 13m 489M - 19.3.0-5103-gf36283e7 48ed0a2cf89d 3cbeaaa88241 2024-09-21T04:14:48.856 INFO:teuthology.orchestra.run.smithi089.stdout:mgr.smithi195.cgotzm smithi195 *:8443,9283,8765 running (72s) 24s ago 11m 452M - 19.3.0-5103-gf36283e7 48ed0a2cf89d d3734574cf66 2024-09-21T04:14:48.856 INFO:teuthology.orchestra.run.smithi089.stdout:mon.smithi089 smithi089 running (54s) 37s ago 13m 31.9M 2048M 19.3.0-5103-gf36283e7 48ed0a2cf89d 7dd34c8a6f21 2024-09-21T04:14:48.856 INFO:teuthology.orchestra.run.smithi089.stdout:mon.smithi195 smithi195 running (29s) 24s ago 11m 23.7M 2048M 19.3.0-5103-gf36283e7 48ed0a2cf89d 4255937444e7 2024-09-21T04:14:48.856 INFO:teuthology.orchestra.run.smithi089.stdout:nfs.foo.0.0.smithi089.dcjmqc smithi089 *:2049 running (5m) 37s ago 5m 274M - 4.0 e1d6a67b021e 98550f94e773 2024-09-21T04:14:48.856 INFO:teuthology.orchestra.run.smithi089.stdout:nfs.foo.1.0.smithi195.vhoayp smithi195 *:2049 running (5m) 24s ago 5m 88.0M - 4.0 e1d6a67b021e c60a4040130a 2024-09-21T04:14:48.856 INFO:teuthology.orchestra.run.smithi089.stdout:node-exporter.smithi089 smithi089 *:9100 running (2m) 37s ago 11m 9.96M - 1.7.0 72c9c2088986 7ef0d1588475 2024-09-21T04:14:48.856 INFO:teuthology.orchestra.run.smithi089.stdout:node-exporter.smithi195 smithi195 *:9100 running (2m) 24s ago 11m 10.4M - 1.7.0 72c9c2088986 61eafd61bc6c 2024-09-21T04:14:48.856 INFO:teuthology.orchestra.run.smithi089.stdout:osd.0 smithi089 running (10m) 37s ago 10m 472M 1935M 17.2.0 e1d6a67b021e db7c0e5020e7 2024-09-21T04:14:48.856 INFO:teuthology.orchestra.run.smithi089.stdout:osd.1 smithi089 running (10m) 37s ago 10m 397M 1935M 17.2.0 e1d6a67b021e f9ec697d08d0 2024-09-21T04:14:48.856 INFO:teuthology.orchestra.run.smithi089.stdout:osd.2 smithi089 running (9m) 37s ago 9m 326M 1935M 17.2.0 e1d6a67b021e cfa000948859 2024-09-21T04:14:48.856 INFO:teuthology.orchestra.run.smithi089.stdout:osd.3 smithi089 running (9m) 37s ago 9m 427M 1935M 17.2.0 e1d6a67b021e 31c610ed5815 2024-09-21T04:14:48.856 INFO:teuthology.orchestra.run.smithi089.stdout:osd.4 smithi195 running (8m) 24s ago 8m 477M 2704M 17.2.0 e1d6a67b021e f00e18a1f220 2024-09-21T04:14:48.857 INFO:teuthology.orchestra.run.smithi089.stdout:osd.5 smithi195 running (8m) 24s ago 8m 510M 2704M 17.2.0 e1d6a67b021e cecfd6ff1eee 2024-09-21T04:14:48.857 INFO:teuthology.orchestra.run.smithi089.stdout:osd.6 smithi195 running (7m) 24s ago 7m 422M 2704M 17.2.0 e1d6a67b021e 0d7db362cc23 2024-09-21T04:14:48.857 INFO:teuthology.orchestra.run.smithi089.stdout:osd.7 smithi195 running (7m) 24s ago 7m 561M 2704M 17.2.0 e1d6a67b021e 0109b3fea188 2024-09-21T04:14:48.857 INFO:teuthology.orchestra.run.smithi089.stdout:prometheus.smithi089 smithi089 *:9095 running (78s) 37s ago 10m 46.1M - 2.51.0 1d3b7f56885b 8c9f309a6459 2024-09-21T04:14:49.279 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:49 smithi195 ceph-mon[49435]: from='client.44101 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T04:14:49.279 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:49 smithi195 ceph-mon[49435]: pgmap v26: 97 pgs: 97 active+clean; 338 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 7.0 KiB/s rd, 6.4 MiB/s wr, 658 op/s 2024-09-21T04:14:49.279 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:49 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:14:49.279 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:49 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:14:49.279 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:49 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "auth get-or-create", "entity": "client.crash.smithi195", "caps": ["mon", "profile crash", "mgr", "profile crash"]} : dispatch 2024-09-21T04:14:49.280 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:49 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T04:14:49.327 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:14:49 smithi089 ceph-mon[64076]: from='client.44101 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T04:14:49.328 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:14:49 smithi089 ceph-mon[64076]: pgmap v26: 97 pgs: 97 active+clean; 338 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 7.0 KiB/s rd, 6.4 MiB/s wr, 658 op/s 2024-09-21T04:14:49.328 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:14:49 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:14:49.328 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:14:49 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:14:49.328 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:14:49 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "auth get-or-create", "entity": "client.crash.smithi195", "caps": ["mon", "profile crash", "mgr", "profile crash"]} : dispatch 2024-09-21T04:14:49.328 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:14:49 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T04:14:49.339 INFO:teuthology.orchestra.run.smithi089.stdout: 5 144985 10.40 MB/sec execute 208 sec latency 28.304 ms 2024-09-21T04:14:49.476 INFO:teuthology.orchestra.run.smithi089.stdout:{ 2024-09-21T04:14:49.476 INFO:teuthology.orchestra.run.smithi089.stdout: "mon": { 2024-09-21T04:14:49.476 INFO:teuthology.orchestra.run.smithi089.stdout: "ceph version 19.3.0-5103-gf36283e7 (f36283e7c91cac1c01cb9a7c4ff21bd697dbd4f2) squid (dev)": 2 2024-09-21T04:14:49.476 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2024-09-21T04:14:49.476 INFO:teuthology.orchestra.run.smithi089.stdout: "mgr": { 2024-09-21T04:14:49.476 INFO:teuthology.orchestra.run.smithi089.stdout: "ceph version 19.3.0-5103-gf36283e7 (f36283e7c91cac1c01cb9a7c4ff21bd697dbd4f2) squid (dev)": 2 2024-09-21T04:14:49.476 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2024-09-21T04:14:49.476 INFO:teuthology.orchestra.run.smithi089.stdout: "osd": { 2024-09-21T04:14:49.476 INFO:teuthology.orchestra.run.smithi089.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-09-21T04:14:49.476 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2024-09-21T04:14:49.476 INFO:teuthology.orchestra.run.smithi089.stdout: "mds": { 2024-09-21T04:14:49.476 INFO:teuthology.orchestra.run.smithi089.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-21T04:14:49.477 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2024-09-21T04:14:49.477 INFO:teuthology.orchestra.run.smithi089.stdout: "overall": { 2024-09-21T04:14:49.477 INFO:teuthology.orchestra.run.smithi089.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 10, 2024-09-21T04:14:49.477 INFO:teuthology.orchestra.run.smithi089.stdout: "ceph version 19.3.0-5103-gf36283e7 (f36283e7c91cac1c01cb9a7c4ff21bd697dbd4f2) squid (dev)": 4 2024-09-21T04:14:49.477 INFO:teuthology.orchestra.run.smithi089.stdout: } 2024-09-21T04:14:49.477 INFO:teuthology.orchestra.run.smithi089.stdout:} 2024-09-21T04:14:49.989 INFO:teuthology.orchestra.run.smithi089.stdout:{ 2024-09-21T04:14:49.990 INFO:teuthology.orchestra.run.smithi089.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:2195e5f178be0a9390728e2a14088236d8fe6f82635f5c84418fe37a04457e00", 2024-09-21T04:14:49.990 INFO:teuthology.orchestra.run.smithi089.stdout: "in_progress": true, 2024-09-21T04:14:49.990 INFO:teuthology.orchestra.run.smithi089.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-21T04:14:49.990 INFO:teuthology.orchestra.run.smithi089.stdout: "services_complete": [ 2024-09-21T04:14:49.990 INFO:teuthology.orchestra.run.smithi089.stdout: "mgr", 2024-09-21T04:14:49.990 INFO:teuthology.orchestra.run.smithi089.stdout: "mon" 2024-09-21T04:14:49.990 INFO:teuthology.orchestra.run.smithi089.stdout: ], 2024-09-21T04:14:49.990 INFO:teuthology.orchestra.run.smithi089.stdout: "progress": "4/23 daemons upgraded", 2024-09-21T04:14:49.990 INFO:teuthology.orchestra.run.smithi089.stdout: "message": "Currently upgrading mon daemons", 2024-09-21T04:14:49.990 INFO:teuthology.orchestra.run.smithi089.stdout: "is_paused": false 2024-09-21T04:14:49.990 INFO:teuthology.orchestra.run.smithi089.stdout:} 2024-09-21T04:14:50.144 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:14:50 smithi089 ceph-mon[64076]: from='client.34160 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T04:14:50.144 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:14:50 smithi089 ceph-mon[64076]: Reconfiguring crash.smithi195 (monmap changed)... 2024-09-21T04:14:50.144 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:14:50 smithi089 ceph-mon[64076]: Reconfiguring daemon crash.smithi195 on smithi195 2024-09-21T04:14:50.144 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:14:50 smithi089 ceph-mon[64076]: from='client.44107 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T04:14:50.144 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:14:50 smithi089 ceph-mon[64076]: from='client.? 172.21.15.89:0/4200113303' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T04:14:50.339 INFO:teuthology.orchestra.run.smithi089.stdout: 5 145433 10.43 MB/sec execute 209 sec latency 44.315 ms 2024-09-21T04:14:50.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:50 smithi195 ceph-mon[49435]: from='client.34160 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T04:14:50.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:50 smithi195 ceph-mon[49435]: Reconfiguring crash.smithi195 (monmap changed)... 2024-09-21T04:14:50.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:50 smithi195 ceph-mon[49435]: Reconfiguring daemon crash.smithi195 on smithi195 2024-09-21T04:14:50.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:50 smithi195 ceph-mon[49435]: from='client.44107 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T04:14:50.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:50 smithi195 ceph-mon[49435]: from='client.? 172.21.15.89:0/4200113303' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T04:14:50.596 INFO:teuthology.orchestra.run.smithi089.stdout:HEALTH_OK 2024-09-21T04:14:51.151 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:14:51 smithi089 ceph-mon[64076]: pgmap v27: 97 pgs: 97 active+clean; 328 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 12 KiB/s rd, 8.8 MiB/s wr, 858 op/s 2024-09-21T04:14:51.151 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:14:51 smithi089 ceph-mon[64076]: from='client.34172 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T04:14:51.151 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:14:51 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:14:51.151 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:14:51 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:14:51.151 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:14:51 smithi089 ceph-mon[64076]: Reconfiguring mgr.smithi195.cgotzm (monmap changed)... 2024-09-21T04:14:51.151 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:14:51 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "auth get-or-create", "entity": "mgr.smithi195.cgotzm", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]} : dispatch 2024-09-21T04:14:51.151 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:14:51 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "mgr services"} : dispatch 2024-09-21T04:14:51.151 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:14:51 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T04:14:51.151 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:14:51 smithi089 ceph-mon[64076]: Reconfiguring daemon mgr.smithi195.cgotzm on smithi195 2024-09-21T04:14:51.151 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:14:51 smithi089 ceph-mon[64076]: from='client.? 172.21.15.89:0/2562804385' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-21T04:14:51.339 INFO:teuthology.orchestra.run.smithi089.stdout: 5 145736 10.44 MB/sec execute 210 sec latency 37.748 ms 2024-09-21T04:14:51.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:51 smithi195 ceph-mon[49435]: pgmap v27: 97 pgs: 97 active+clean; 328 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 12 KiB/s rd, 8.8 MiB/s wr, 858 op/s 2024-09-21T04:14:51.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:51 smithi195 ceph-mon[49435]: from='client.34172 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T04:14:51.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:51 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:14:51.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:51 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:14:51.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:51 smithi195 ceph-mon[49435]: Reconfiguring mgr.smithi195.cgotzm (monmap changed)... 2024-09-21T04:14:51.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:51 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "auth get-or-create", "entity": "mgr.smithi195.cgotzm", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]} : dispatch 2024-09-21T04:14:51.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:51 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "mgr services"} : dispatch 2024-09-21T04:14:51.610 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:51 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T04:14:51.610 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:51 smithi195 ceph-mon[49435]: Reconfiguring daemon mgr.smithi195.cgotzm on smithi195 2024-09-21T04:14:51.610 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:51 smithi195 ceph-mon[49435]: from='client.? 172.21.15.89:0/2562804385' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-21T04:14:52.339 INFO:teuthology.orchestra.run.smithi089.stdout: 5 146093 10.40 MB/sec execute 211 sec latency 48.364 ms 2024-09-21T04:14:52.928 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:14:52 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:14:52.928 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:14:52 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:14:52.928 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:14:52 smithi089 ceph-mon[64076]: Reconfiguring mon.smithi195 (monmap changed)... 2024-09-21T04:14:52.928 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:14:52 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-09-21T04:14:52.928 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:14:52 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-09-21T04:14:52.928 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:14:52 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T04:14:52.928 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:14:52 smithi089 ceph-mon[64076]: Reconfiguring daemon mon.smithi195 on smithi195 2024-09-21T04:14:52.928 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:14:52 smithi089 ceph-mon[64076]: pgmap v28: 97 pgs: 97 active+clean; 328 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 9.5 KiB/s rd, 5.6 MiB/s wr, 595 op/s 2024-09-21T04:14:53.109 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:52 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:14:53.110 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:52 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:14:53.110 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:52 smithi195 ceph-mon[49435]: Reconfiguring mon.smithi195 (monmap changed)... 2024-09-21T04:14:53.110 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:52 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-09-21T04:14:53.110 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:52 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-09-21T04:14:53.110 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:52 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T04:14:53.110 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:52 smithi195 ceph-mon[49435]: Reconfiguring daemon mon.smithi195 on smithi195 2024-09-21T04:14:53.110 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:52 smithi195 ceph-mon[49435]: pgmap v28: 97 pgs: 97 active+clean; 328 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 9.5 KiB/s rd, 5.6 MiB/s wr, 595 op/s 2024-09-21T04:14:53.340 INFO:teuthology.orchestra.run.smithi089.stdout: 5 146612 10.40 MB/sec execute 212 sec latency 36.562 ms 2024-09-21T04:14:54.339 INFO:teuthology.orchestra.run.smithi089.stdout: 5 147168 10.42 MB/sec execute 213 sec latency 39.213 ms 2024-09-21T04:14:54.692 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:14:54 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:14:54.693 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:14:54 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:14:54.693 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:14:54 smithi089 ceph-mon[64076]: Reconfiguring osd.4 (monmap changed)... 2024-09-21T04:14:54.693 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:14:54 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "auth get", "entity": "osd.4"} : dispatch 2024-09-21T04:14:54.693 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:14:54 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T04:14:54.693 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:14:54 smithi089 ceph-mon[64076]: Reconfiguring daemon osd.4 on smithi195 2024-09-21T04:14:54.693 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:14:54 smithi089 ceph-mon[64076]: pgmap v29: 97 pgs: 97 active+clean; 342 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 12 KiB/s rd, 8.7 MiB/s wr, 857 op/s 2024-09-21T04:14:54.859 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:54 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:14:54.859 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:54 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:14:54.859 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:54 smithi195 ceph-mon[49435]: Reconfiguring osd.4 (monmap changed)... 2024-09-21T04:14:54.859 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:54 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "auth get", "entity": "osd.4"} : dispatch 2024-09-21T04:14:54.859 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:54 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T04:14:54.859 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:54 smithi195 ceph-mon[49435]: Reconfiguring daemon osd.4 on smithi195 2024-09-21T04:14:54.859 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:54 smithi195 ceph-mon[49435]: pgmap v29: 97 pgs: 97 active+clean; 342 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 12 KiB/s rd, 8.7 MiB/s wr, 857 op/s 2024-09-21T04:14:55.340 INFO:teuthology.orchestra.run.smithi089.stdout: 5 147809 10.42 MB/sec execute 214 sec latency 34.985 ms 2024-09-21T04:14:56.216 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:14:56 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:14:56.216 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:14:56 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:14:56.216 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:14:56 smithi089 ceph-mon[64076]: Reconfiguring osd.5 (monmap changed)... 2024-09-21T04:14:56.216 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:14:56 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "auth get", "entity": "osd.5"} : dispatch 2024-09-21T04:14:56.216 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:14:56 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T04:14:56.216 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:14:56 smithi089 ceph-mon[64076]: Reconfiguring daemon osd.5 on smithi195 2024-09-21T04:14:56.339 INFO:teuthology.orchestra.run.smithi089.stdout: 5 148290 10.45 MB/sec execute 215 sec latency 53.003 ms 2024-09-21T04:14:56.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:56 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:14:56.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:56 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:14:56.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:56 smithi195 ceph-mon[49435]: Reconfiguring osd.5 (monmap changed)... 2024-09-21T04:14:56.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:56 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "auth get", "entity": "osd.5"} : dispatch 2024-09-21T04:14:56.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:56 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T04:14:56.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:56 smithi195 ceph-mon[49435]: Reconfiguring daemon osd.5 on smithi195 2024-09-21T04:14:57.230 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:14:57 smithi089 ceph-mon[64076]: pgmap v30: 97 pgs: 97 active+clean; 342 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 7.9 KiB/s rd, 6.4 MiB/s wr, 611 op/s 2024-09-21T04:14:57.230 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:14:57 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:14:57.230 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:14:57 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:14:57.230 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:14:57 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "auth get", "entity": "osd.6"} : dispatch 2024-09-21T04:14:57.230 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:14:57 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T04:14:57.340 INFO:teuthology.orchestra.run.smithi089.stdout: 5 148877 10.52 MB/sec execute 216 sec latency 23.669 ms 2024-09-21T04:14:57.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:57 smithi195 ceph-mon[49435]: pgmap v30: 97 pgs: 97 active+clean; 342 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 7.9 KiB/s rd, 6.4 MiB/s wr, 611 op/s 2024-09-21T04:14:57.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:57 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:14:57.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:57 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:14:57.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:57 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "auth get", "entity": "osd.6"} : dispatch 2024-09-21T04:14:57.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:57 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T04:14:58.243 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:14:58 smithi089 ceph-mon[64076]: Reconfiguring osd.6 (monmap changed)... 2024-09-21T04:14:58.244 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:14:58 smithi089 ceph-mon[64076]: Reconfiguring daemon osd.6 on smithi195 2024-09-21T04:14:58.340 INFO:teuthology.orchestra.run.smithi089.stdout: 5 149209 10.54 MB/sec execute 217 sec latency 58.803 ms 2024-09-21T04:14:58.458 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:58 smithi195 ceph-mon[49435]: Reconfiguring osd.6 (monmap changed)... 2024-09-21T04:14:58.459 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:58 smithi195 ceph-mon[49435]: Reconfiguring daemon osd.6 on smithi195 2024-09-21T04:14:59.208 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:59 smithi195 ceph-mon[49435]: pgmap v31: 97 pgs: 97 active+clean; 341 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 7.9 KiB/s rd, 6.7 MiB/s wr, 624 op/s 2024-09-21T04:14:59.208 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:59 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:14:59.208 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:59 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:14:59.208 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:59 smithi195 ceph-mon[49435]: Reconfiguring osd.7 (monmap changed)... 2024-09-21T04:14:59.208 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:59 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "auth get", "entity": "osd.7"} : dispatch 2024-09-21T04:14:59.208 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:59 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T04:14:59.208 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:14:59 smithi195 ceph-mon[49435]: Reconfiguring daemon osd.7 on smithi195 2024-09-21T04:14:59.255 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:14:59 smithi089 ceph-mon[64076]: pgmap v31: 97 pgs: 97 active+clean; 341 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 7.9 KiB/s rd, 6.7 MiB/s wr, 624 op/s 2024-09-21T04:14:59.255 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:14:59 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:14:59.255 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:14:59 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:14:59.255 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:14:59 smithi089 ceph-mon[64076]: Reconfiguring osd.7 (monmap changed)... 2024-09-21T04:14:59.255 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:14:59 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "auth get", "entity": "osd.7"} : dispatch 2024-09-21T04:14:59.255 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:14:59 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T04:14:59.255 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:14:59 smithi089 ceph-mon[64076]: Reconfiguring daemon osd.7 on smithi195 2024-09-21T04:14:59.340 INFO:teuthology.orchestra.run.smithi089.stdout: 5 149398 10.50 MB/sec execute 218 sec latency 100.963 ms 2024-09-21T04:15:00.273 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:15:00 smithi089 ceph-mon[64076]: pgmap v32: 97 pgs: 97 active+clean; 340 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 13 KiB/s rd, 8.8 MiB/s wr, 776 op/s 2024-09-21T04:15:00.340 INFO:teuthology.orchestra.run.smithi089.stdout: 5 149655 10.46 MB/sec execute 219 sec latency 54.569 ms 2024-09-21T04:15:00.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:15:00 smithi195 ceph-mon[49435]: pgmap v32: 97 pgs: 97 active+clean; 340 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 13 KiB/s rd, 8.8 MiB/s wr, 776 op/s 2024-09-21T04:15:01.315 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:15:01 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:15:01.315 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:15:01 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:15:01.315 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:15:01 smithi089 ceph-mon[64076]: Reconfiguring mds.foofs.smithi195.hijxre (monmap changed)... 2024-09-21T04:15:01.315 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:15:01 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "auth get-or-create", "entity": "mds.foofs.smithi195.hijxre", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-09-21T04:15:01.315 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:15:01 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T04:15:01.315 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:15:01 smithi089 ceph-mon[64076]: Reconfiguring daemon mds.foofs.smithi195.hijxre on smithi195 2024-09-21T04:15:01.340 INFO:teuthology.orchestra.run.smithi089.stdout: 5 149900 10.43 MB/sec execute 220 sec latency 55.083 ms 2024-09-21T04:15:01.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:15:01 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:15:01.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:15:01 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:15:01.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:15:01 smithi195 ceph-mon[49435]: Reconfiguring mds.foofs.smithi195.hijxre (monmap changed)... 2024-09-21T04:15:01.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:15:01 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "auth get-or-create", "entity": "mds.foofs.smithi195.hijxre", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-09-21T04:15:01.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:15:01 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T04:15:01.610 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:15:01 smithi195 ceph-mon[49435]: Reconfiguring daemon mds.foofs.smithi195.hijxre on smithi195 2024-09-21T04:15:02.340 INFO:teuthology.orchestra.run.smithi089.stdout: 5 150211 10.42 MB/sec execute 221 sec latency 60.264 ms 2024-09-21T04:15:03.101 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:15:02 smithi089 ceph-mon[64076]: pgmap v33: 97 pgs: 97 active+clean; 340 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 7.4 KiB/s rd, 6.4 MiB/s wr, 576 op/s 2024-09-21T04:15:03.101 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:15:02 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:15:03.101 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:15:02 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:15:03.102 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:15:02 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T04:15:03.102 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:15:02 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "versions"} : dispatch 2024-09-21T04:15:03.102 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:15:02 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "versions"} : dispatch 2024-09-21T04:15:03.102 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:15:02 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:15:03.102 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:15:02 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.smithi089"} : dispatch 2024-09-21T04:15:03.102 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:15:02 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.smithi089"}]': finished 2024-09-21T04:15:03.102 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:15:02 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.smithi195"} : dispatch 2024-09-21T04:15:03.102 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:15:02 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.smithi195"}]': finished 2024-09-21T04:15:03.102 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:15:02 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T04:15:03.340 INFO:teuthology.orchestra.run.smithi089.stdout: 5 150725 10.43 MB/sec execute 222 sec latency 110.374 ms 2024-09-21T04:15:03.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:15:02 smithi195 ceph-mon[49435]: pgmap v33: 97 pgs: 97 active+clean; 340 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 7.4 KiB/s rd, 6.4 MiB/s wr, 576 op/s 2024-09-21T04:15:03.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:15:02 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:15:03.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:15:02 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:15:03.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:15:02 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T04:15:03.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:15:02 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "versions"} : dispatch 2024-09-21T04:15:03.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:15:02 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "versions"} : dispatch 2024-09-21T04:15:03.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:15:02 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:15:03.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:15:02 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.smithi089"} : dispatch 2024-09-21T04:15:03.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:15:02 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.smithi089"}]': finished 2024-09-21T04:15:03.360 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:15:02 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.smithi195"} : dispatch 2024-09-21T04:15:03.360 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:15:02 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.smithi195"}]': finished 2024-09-21T04:15:03.360 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:15:02 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T04:15:04.134 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:15:03 smithi089 ceph-mon[64076]: Upgrade: Setting container_image for all mon 2024-09-21T04:15:04.340 INFO:teuthology.orchestra.run.smithi089.stdout: 5 151240 10.41 MB/sec execute 223 sec latency 48.294 ms 2024-09-21T04:15:04.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:15:03 smithi195 ceph-mon[49435]: Upgrade: Setting container_image for all mon 2024-09-21T04:15:05.147 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:15:04 smithi089 ceph-mon[64076]: pgmap v34: 97 pgs: 97 active+clean; 330 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 8.1 MiB/s wr, 769 op/s 2024-09-21T04:15:05.147 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:15:04 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:15:05.147 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:15:04 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "auth get-or-create", "entity": "client.crash.smithi089", "caps": ["mon", "profile crash", "mgr", "profile crash"]} : dispatch 2024-09-21T04:15:05.147 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:15:04 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T04:15:05.340 INFO:teuthology.orchestra.run.smithi089.stdout: 5 151684 10.44 MB/sec execute 224 sec latency 47.868 ms 2024-09-21T04:15:05.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:15:04 smithi195 ceph-mon[49435]: pgmap v34: 97 pgs: 97 active+clean; 330 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 8.1 MiB/s wr, 769 op/s 2024-09-21T04:15:05.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:15:04 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:15:05.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:15:04 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "auth get-or-create", "entity": "client.crash.smithi089", "caps": ["mon", "profile crash", "mgr", "profile crash"]} : dispatch 2024-09-21T04:15:05.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:15:04 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T04:15:06.165 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:15:05 smithi089 ceph-mon[64076]: Upgrade: Updating crash.smithi089 (1/2) 2024-09-21T04:15:06.166 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:15:05 smithi089 ceph-mon[64076]: Deploying daemon crash.smithi089 on smithi089 2024-09-21T04:15:06.340 INFO:teuthology.orchestra.run.smithi089.stdout: 5 152380 10.52 MB/sec execute 225 sec latency 64.870 ms 2024-09-21T04:15:06.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:15:05 smithi195 ceph-mon[49435]: Upgrade: Updating crash.smithi089 (1/2) 2024-09-21T04:15:06.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:15:05 smithi195 ceph-mon[49435]: Deploying daemon crash.smithi089 on smithi089 2024-09-21T04:15:07.178 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:15:06 smithi089 ceph-mon[64076]: pgmap v35: 97 pgs: 97 active+clean; 330 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 8.8 KiB/s rd, 5.0 MiB/s wr, 508 op/s 2024-09-21T04:15:07.259 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:15:06 smithi195 ceph-mon[49435]: pgmap v35: 97 pgs: 97 active+clean; 330 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 8.8 KiB/s rd, 5.0 MiB/s wr, 508 op/s 2024-09-21T04:15:07.340 INFO:teuthology.orchestra.run.smithi089.stdout: 5 152779 10.54 MB/sec execute 226 sec latency 39.108 ms 2024-09-21T04:15:08.340 INFO:teuthology.orchestra.run.smithi089.stdout: 5 153069 10.51 MB/sec execute 227 sec latency 43.703 ms 2024-09-21T04:15:09.193 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:15:08 smithi089 ceph-mon[64076]: pgmap v36: 97 pgs: 97 active+clean; 334 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 8.8 KiB/s rd, 5.7 MiB/s wr, 586 op/s 2024-09-21T04:15:09.340 INFO:teuthology.orchestra.run.smithi089.stdout: 5 153355 10.47 MB/sec execute 228 sec latency 86.370 ms 2024-09-21T04:15:09.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:15:08 smithi195 ceph-mon[49435]: pgmap v36: 97 pgs: 97 active+clean; 334 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 8.8 KiB/s rd, 5.7 MiB/s wr, 586 op/s 2024-09-21T04:15:10.340 INFO:teuthology.orchestra.run.smithi089.stdout: 5 153886 10.49 MB/sec execute 229 sec latency 42.413 ms 2024-09-21T04:15:11.207 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:15:10 smithi089 ceph-mon[64076]: pgmap v37: 97 pgs: 97 active+clean; 338 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 12 KiB/s rd, 7.9 MiB/s wr, 771 op/s 2024-09-21T04:15:11.207 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:15:10 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:15:11.207 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:15:10 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:15:11.340 INFO:teuthology.orchestra.run.smithi089.stdout: 5 154679 10.50 MB/sec execute 230 sec latency 29.781 ms 2024-09-21T04:15:11.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:15:10 smithi195 ceph-mon[49435]: pgmap v37: 97 pgs: 97 active+clean; 338 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 12 KiB/s rd, 7.9 MiB/s wr, 771 op/s 2024-09-21T04:15:11.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:15:10 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:15:11.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:15:10 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:15:12.340 INFO:teuthology.orchestra.run.smithi089.stdout: 5 155172 10.52 MB/sec execute 231 sec latency 31.874 ms 2024-09-21T04:15:13.249 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:15:13 smithi089 ceph-mon[64076]: pgmap v38: 97 pgs: 97 active+clean; 338 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.8 KiB/s rd, 4.8 MiB/s wr, 469 op/s 2024-09-21T04:15:13.249 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:15:13 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:15:13.249 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:15:13 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "auth get-or-create", "entity": "client.crash.smithi195", "caps": ["mon", "profile crash", "mgr", "profile crash"]} : dispatch 2024-09-21T04:15:13.249 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:15:13 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T04:15:13.293 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:15:13 smithi195 ceph-mon[49435]: pgmap v38: 97 pgs: 97 active+clean; 338 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.8 KiB/s rd, 4.8 MiB/s wr, 469 op/s 2024-09-21T04:15:13.294 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:15:13 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:15:13.294 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:15:13 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "auth get-or-create", "entity": "client.crash.smithi195", "caps": ["mon", "profile crash", "mgr", "profile crash"]} : dispatch 2024-09-21T04:15:13.294 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:15:13 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T04:15:13.340 INFO:teuthology.orchestra.run.smithi089.stdout: 5 155676 10.58 MB/sec execute 232 sec latency 41.765 ms 2024-09-21T04:15:14.024 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:15:14 smithi089 ceph-mon[64076]: Upgrade: Updating crash.smithi195 (2/2) 2024-09-21T04:15:14.024 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:15:14 smithi089 ceph-mon[64076]: Deploying daemon crash.smithi195 on smithi195 2024-09-21T04:15:14.340 INFO:teuthology.orchestra.run.smithi089.stdout: 5 156133 10.61 MB/sec execute 233 sec latency 30.413 ms 2024-09-21T04:15:14.351 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:15:14 smithi195 ceph-mon[49435]: Upgrade: Updating crash.smithi195 (2/2) 2024-09-21T04:15:14.351 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:15:14 smithi195 ceph-mon[49435]: Deploying daemon crash.smithi195 on smithi195 2024-09-21T04:15:15.032 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:15:15 smithi089 ceph-mon[64076]: pgmap v39: 97 pgs: 97 active+clean; 334 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 12 KiB/s rd, 7.6 MiB/s wr, 779 op/s 2024-09-21T04:15:15.257 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:15:15 smithi195 ceph-mon[49435]: pgmap v39: 97 pgs: 97 active+clean; 334 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 12 KiB/s rd, 7.6 MiB/s wr, 779 op/s 2024-09-21T04:15:15.340 INFO:teuthology.orchestra.run.smithi089.stdout: 5 156606 10.60 MB/sec execute 234 sec latency 28.995 ms 2024-09-21T04:15:16.340 INFO:teuthology.orchestra.run.smithi089.stdout: 5 157170 10.59 MB/sec execute 235 sec latency 29.479 ms 2024-09-21T04:15:17.184 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:15:17 smithi089 ceph-mon[64076]: pgmap v40: 97 pgs: 97 active+clean; 334 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.5 KiB/s rd, 5.9 MiB/s wr, 585 op/s 2024-09-21T04:15:17.184 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:15:17 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:15:17.184 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:15:17 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:15:17.184 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:15:17 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T04:15:17.307 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:15:17 smithi195 ceph-mon[49435]: pgmap v40: 97 pgs: 97 active+clean; 334 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.5 KiB/s rd, 5.9 MiB/s wr, 585 op/s 2024-09-21T04:15:17.307 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:15:17 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:15:17.307 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:15:17 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:15:17.307 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:15:17 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T04:15:17.340 INFO:teuthology.orchestra.run.smithi089.stdout: 5 157838 10.63 MB/sec execute 236 sec latency 71.551 ms 2024-09-21T04:15:18.205 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:15:18 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T04:15:18.340 INFO:teuthology.orchestra.run.smithi089.stdout: 5 158272 10.60 MB/sec execute 237 sec latency 87.240 ms 2024-09-21T04:15:18.358 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:15:18 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T04:15:19.220 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:15:19 smithi089 ceph-mon[64076]: pgmap v41: 97 pgs: 97 active+clean; 328 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.5 KiB/s rd, 6.4 MiB/s wr, 605 op/s 2024-09-21T04:15:19.340 INFO:teuthology.orchestra.run.smithi089.stdout: 5 158621 10.59 MB/sec execute 238 sec latency 139.109 ms 2024-09-21T04:15:19.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:15:19 smithi195 ceph-mon[49435]: pgmap v41: 97 pgs: 97 active+clean; 328 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.5 KiB/s rd, 6.4 MiB/s wr, 605 op/s 2024-09-21T04:15:20.341 INFO:teuthology.orchestra.run.smithi089.stdout: 5 158994 10.62 MB/sec execute 239 sec latency 38.508 ms 2024-09-21T04:15:21.095 INFO:teuthology.orchestra.run.smithi089.stdout:true 2024-09-21T04:15:21.272 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:15:21 smithi089 ceph-mon[64076]: pgmap v42: 97 pgs: 97 active+clean; 331 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 12 KiB/s rd, 8.4 MiB/s wr, 825 op/s 2024-09-21T04:15:21.346 INFO:teuthology.orchestra.run.smithi089.stdout: 5 159418 10.65 MB/sec execute 240 sec latency 54.105 ms 2024-09-21T04:15:21.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:15:21 smithi195 ceph-mon[49435]: pgmap v42: 97 pgs: 97 active+clean; 331 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 12 KiB/s rd, 8.4 MiB/s wr, 825 op/s 2024-09-21T04:15:22.051 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:15:22 smithi089 ceph-mon[64076]: from='client.34180 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T04:15:22.051 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:15:22 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:15:22.051 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:15:22 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:15:22.152 INFO:teuthology.orchestra.run.smithi089.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-21T04:15:22.152 INFO:teuthology.orchestra.run.smithi089.stdout:alertmanager.smithi089 smithi089 *:9093,9094 running (3m) 70s ago 12m 19.5M - 0.27.0 11f11916f8cd 65dfcbcafee1 2024-09-21T04:15:22.152 INFO:teuthology.orchestra.run.smithi089.stdout:crash.smithi089 smithi089 starting - - - - 2024-09-21T04:15:22.152 INFO:teuthology.orchestra.run.smithi089.stdout:crash.smithi195 smithi195 running (6s) 0s ago 11m 6891k - 19.3.0-5103-gf36283e7 48ed0a2cf89d 78ce05a4e9ab 2024-09-21T04:15:22.152 INFO:teuthology.orchestra.run.smithi089.stdout:grafana.smithi089 smithi089 *:3000 running (3m) 70s ago 12m 45.0M - dad864ee21e9 ec4d6738bce7 2024-09-21T04:15:22.152 INFO:teuthology.orchestra.run.smithi089.stdout:mds.foofs.smithi089.osufws smithi089 running (6m) 70s ago 6m 37.9M - 17.2.0 e1d6a67b021e 664d41859bea 2024-09-21T04:15:22.152 INFO:teuthology.orchestra.run.smithi089.stdout:mds.foofs.smithi195.hijxre smithi195 running (6m) 0s ago 6m 16.7M - 17.2.0 e1d6a67b021e 7bc0c6542d12 2024-09-21T04:15:22.153 INFO:teuthology.orchestra.run.smithi089.stdout:mgr.smithi089.srlyxf smithi089 *:8443,9283,8765 running (2m) 70s ago 13m 489M - 19.3.0-5103-gf36283e7 48ed0a2cf89d 3cbeaaa88241 2024-09-21T04:15:22.153 INFO:teuthology.orchestra.run.smithi089.stdout:mgr.smithi195.cgotzm smithi195 *:8443,9283,8765 running (105s) 0s ago 11m 454M - 19.3.0-5103-gf36283e7 48ed0a2cf89d d3734574cf66 2024-09-21T04:15:22.153 INFO:teuthology.orchestra.run.smithi089.stdout:mon.smithi089 smithi089 running (87s) 70s ago 13m 31.9M 2048M 19.3.0-5103-gf36283e7 48ed0a2cf89d 7dd34c8a6f21 2024-09-21T04:15:22.153 INFO:teuthology.orchestra.run.smithi089.stdout:mon.smithi195 smithi195 running (63s) 0s ago 11m 36.7M 2048M 19.3.0-5103-gf36283e7 48ed0a2cf89d 4255937444e7 2024-09-21T04:15:22.153 INFO:teuthology.orchestra.run.smithi089.stdout:nfs.foo.0.0.smithi089.dcjmqc smithi089 *:2049 running (6m) 70s ago 6m 274M - 4.0 e1d6a67b021e 98550f94e773 2024-09-21T04:15:22.153 INFO:teuthology.orchestra.run.smithi089.stdout:nfs.foo.1.0.smithi195.vhoayp smithi195 *:2049 running (6m) 0s ago 6m 88.0M - 4.0 e1d6a67b021e c60a4040130a 2024-09-21T04:15:22.153 INFO:teuthology.orchestra.run.smithi089.stdout:node-exporter.smithi089 smithi089 *:9100 running (3m) 70s ago 12m 9.96M - 1.7.0 72c9c2088986 7ef0d1588475 2024-09-21T04:15:22.153 INFO:teuthology.orchestra.run.smithi089.stdout:node-exporter.smithi195 smithi195 *:9100 running (2m) 0s ago 11m 10.7M - 1.7.0 72c9c2088986 61eafd61bc6c 2024-09-21T04:15:22.153 INFO:teuthology.orchestra.run.smithi089.stdout:osd.0 smithi089 running (11m) 70s ago 11m 472M 1935M 17.2.0 e1d6a67b021e db7c0e5020e7 2024-09-21T04:15:22.153 INFO:teuthology.orchestra.run.smithi089.stdout:osd.1 smithi089 running (10m) 70s ago 10m 397M 1935M 17.2.0 e1d6a67b021e f9ec697d08d0 2024-09-21T04:15:22.153 INFO:teuthology.orchestra.run.smithi089.stdout:osd.2 smithi089 running (10m) 70s ago 10m 326M 1935M 17.2.0 e1d6a67b021e cfa000948859 2024-09-21T04:15:22.153 INFO:teuthology.orchestra.run.smithi089.stdout:osd.3 smithi089 running (9m) 70s ago 9m 427M 1935M 17.2.0 e1d6a67b021e 31c610ed5815 2024-09-21T04:15:22.153 INFO:teuthology.orchestra.run.smithi089.stdout:osd.4 smithi195 running (9m) 0s ago 9m 524M 2704M 17.2.0 e1d6a67b021e f00e18a1f220 2024-09-21T04:15:22.153 INFO:teuthology.orchestra.run.smithi089.stdout:osd.5 smithi195 running (8m) 0s ago 8m 318M 2704M 17.2.0 e1d6a67b021e cecfd6ff1eee 2024-09-21T04:15:22.153 INFO:teuthology.orchestra.run.smithi089.stdout:osd.6 smithi195 running (8m) 0s ago 8m 510M 2704M 17.2.0 e1d6a67b021e 0d7db362cc23 2024-09-21T04:15:22.154 INFO:teuthology.orchestra.run.smithi089.stdout:osd.7 smithi195 running (8m) 0s ago 8m 387M 2704M 17.2.0 e1d6a67b021e 0109b3fea188 2024-09-21T04:15:22.154 INFO:teuthology.orchestra.run.smithi089.stdout:prometheus.smithi089 smithi089 *:9095 running (111s) 70s ago 11m 46.1M - 2.51.0 1d3b7f56885b 8c9f309a6459 2024-09-21T04:15:22.346 INFO:teuthology.orchestra.run.smithi089.stdout: 5 159750 10.67 MB/sec execute 241 sec latency 45.951 ms 2024-09-21T04:15:22.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:15:22 smithi195 ceph-mon[49435]: from='client.34180 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T04:15:22.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:15:22 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:15:22.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:15:22 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:15:22.811 INFO:teuthology.orchestra.run.smithi089.stdout:{ 2024-09-21T04:15:22.811 INFO:teuthology.orchestra.run.smithi089.stdout: "mon": { 2024-09-21T04:15:22.811 INFO:teuthology.orchestra.run.smithi089.stdout: "ceph version 19.3.0-5103-gf36283e7 (f36283e7c91cac1c01cb9a7c4ff21bd697dbd4f2) squid (dev)": 2 2024-09-21T04:15:22.811 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2024-09-21T04:15:22.811 INFO:teuthology.orchestra.run.smithi089.stdout: "mgr": { 2024-09-21T04:15:22.811 INFO:teuthology.orchestra.run.smithi089.stdout: "ceph version 19.3.0-5103-gf36283e7 (f36283e7c91cac1c01cb9a7c4ff21bd697dbd4f2) squid (dev)": 2 2024-09-21T04:15:22.811 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2024-09-21T04:15:22.811 INFO:teuthology.orchestra.run.smithi089.stdout: "osd": { 2024-09-21T04:15:22.811 INFO:teuthology.orchestra.run.smithi089.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-09-21T04:15:22.811 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2024-09-21T04:15:22.811 INFO:teuthology.orchestra.run.smithi089.stdout: "mds": { 2024-09-21T04:15:22.811 INFO:teuthology.orchestra.run.smithi089.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-21T04:15:22.812 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2024-09-21T04:15:22.812 INFO:teuthology.orchestra.run.smithi089.stdout: "overall": { 2024-09-21T04:15:22.812 INFO:teuthology.orchestra.run.smithi089.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 10, 2024-09-21T04:15:22.812 INFO:teuthology.orchestra.run.smithi089.stdout: "ceph version 19.3.0-5103-gf36283e7 (f36283e7c91cac1c01cb9a7c4ff21bd697dbd4f2) squid (dev)": 4 2024-09-21T04:15:22.812 INFO:teuthology.orchestra.run.smithi089.stdout: } 2024-09-21T04:15:22.812 INFO:teuthology.orchestra.run.smithi089.stdout:} 2024-09-21T04:15:23.070 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:15:23 smithi089 ceph-mon[64076]: from='client.34184 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T04:15:23.070 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:15:23 smithi089 ceph-mon[64076]: pgmap v43: 97 pgs: 97 active+clean; 331 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 9.6 KiB/s rd, 5.9 MiB/s wr, 627 op/s 2024-09-21T04:15:23.070 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:15:23 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:15:23.070 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:15:23 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:15:23.070 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:15:23 smithi089 ceph-mon[64076]: from='client.? 172.21.15.89:0/1358297545' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T04:15:23.346 INFO:teuthology.orchestra.run.smithi089.stdout: 5 159963 10.64 MB/sec execute 242 sec latency 40.012 ms 2024-09-21T04:15:23.347 INFO:teuthology.orchestra.run.smithi089.stdout:{ 2024-09-21T04:15:23.347 INFO:teuthology.orchestra.run.smithi089.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:2195e5f178be0a9390728e2a14088236d8fe6f82635f5c84418fe37a04457e00", 2024-09-21T04:15:23.347 INFO:teuthology.orchestra.run.smithi089.stdout: "in_progress": true, 2024-09-21T04:15:23.347 INFO:teuthology.orchestra.run.smithi089.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-21T04:15:23.347 INFO:teuthology.orchestra.run.smithi089.stdout: "services_complete": [ 2024-09-21T04:15:23.347 INFO:teuthology.orchestra.run.smithi089.stdout: "mgr", 2024-09-21T04:15:23.348 INFO:teuthology.orchestra.run.smithi089.stdout: "mon" 2024-09-21T04:15:23.348 INFO:teuthology.orchestra.run.smithi089.stdout: ], 2024-09-21T04:15:23.348 INFO:teuthology.orchestra.run.smithi089.stdout: "progress": "5/23 daemons upgraded", 2024-09-21T04:15:23.348 INFO:teuthology.orchestra.run.smithi089.stdout: "message": "Currently upgrading crash daemons", 2024-09-21T04:15:23.348 INFO:teuthology.orchestra.run.smithi089.stdout: "is_paused": false 2024-09-21T04:15:23.348 INFO:teuthology.orchestra.run.smithi089.stdout:} 2024-09-21T04:15:23.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:15:23 smithi195 ceph-mon[49435]: from='client.34184 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T04:15:23.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:15:23 smithi195 ceph-mon[49435]: pgmap v43: 97 pgs: 97 active+clean; 331 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 9.6 KiB/s rd, 5.9 MiB/s wr, 627 op/s 2024-09-21T04:15:23.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:15:23 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:15:23.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:15:23 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:15:23.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:15:23 smithi195 ceph-mon[49435]: from='client.? 172.21.15.89:0/1358297545' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T04:15:24.056 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:15:24 smithi195 ceph-mon[49435]: from='client.34188 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T04:15:24.094 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:15:24 smithi089 ceph-mon[64076]: from='client.34188 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T04:15:24.119 INFO:teuthology.orchestra.run.smithi089.stdout:HEALTH_OK 2024-09-21T04:15:24.346 INFO:teuthology.orchestra.run.smithi089.stdout: 5 160206 10.61 MB/sec execute 243 sec latency 40.902 ms 2024-09-21T04:15:25.107 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:15:25 smithi089 ceph-mon[64076]: from='client.44139 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T04:15:25.107 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:15:25 smithi089 ceph-mon[64076]: pgmap v44: 97 pgs: 97 active+clean; 342 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 12 KiB/s rd, 8.5 MiB/s wr, 839 op/s 2024-09-21T04:15:25.107 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:15:25 smithi089 ceph-mon[64076]: from='client.? 172.21.15.89:0/521364756' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-21T04:15:25.107 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:15:25 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:15:25.107 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:15:25 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:15:25.346 INFO:teuthology.orchestra.run.smithi089.stdout: 5 160463 10.57 MB/sec execute 244 sec latency 86.408 ms 2024-09-21T04:15:25.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:15:25 smithi195 ceph-mon[49435]: from='client.44139 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T04:15:25.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:15:25 smithi195 ceph-mon[49435]: pgmap v44: 97 pgs: 97 active+clean; 342 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 12 KiB/s rd, 8.5 MiB/s wr, 839 op/s 2024-09-21T04:15:25.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:15:25 smithi195 ceph-mon[49435]: from='client.? 172.21.15.89:0/521364756' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-21T04:15:25.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:15:25 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:15:25.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:15:25 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:15:26.347 INFO:teuthology.orchestra.run.smithi089.stdout: 5 160716 10.56 MB/sec execute 245 sec latency 41.189 ms 2024-09-21T04:15:26.659 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:15:26 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:15:26.659 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:15:26 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:15:26.659 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:15:26 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:15:26.659 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:15:26 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:15:26.659 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:15:26 smithi089 ceph-mon[64076]: pgmap v45: 97 pgs: 97 active+clean; 342 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 7.2 KiB/s rd, 5.7 MiB/s wr, 529 op/s 2024-09-21T04:15:26.859 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:15:26 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:15:26.859 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:15:26 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:15:26.859 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:15:26 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:15:26.859 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:15:26 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:15:26.859 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:15:26 smithi195 ceph-mon[49435]: pgmap v45: 97 pgs: 97 active+clean; 342 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 7.2 KiB/s rd, 5.7 MiB/s wr, 529 op/s 2024-09-21T04:15:27.346 INFO:teuthology.orchestra.run.smithi089.stdout: 5 160976 10.54 MB/sec execute 246 sec latency 74.235 ms 2024-09-21T04:15:28.346 INFO:teuthology.orchestra.run.smithi089.stdout: 5 161349 10.54 MB/sec execute 247 sec latency 97.087 ms 2024-09-21T04:15:29.175 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:15:28 smithi089 ceph-mon[64076]: pgmap v46: 97 pgs: 97 active+clean; 344 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 7.2 KiB/s rd, 6.2 MiB/s wr, 613 op/s 2024-09-21T04:15:29.347 INFO:teuthology.orchestra.run.smithi089.stdout: 5 161780 10.51 MB/sec execute 248 sec latency 34.762 ms 2024-09-21T04:15:29.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:15:28 smithi195 ceph-mon[49435]: pgmap v46: 97 pgs: 97 active+clean; 344 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 7.2 KiB/s rd, 6.2 MiB/s wr, 613 op/s 2024-09-21T04:15:30.347 INFO:teuthology.orchestra.run.smithi089.stdout: 5 162242 10.53 MB/sec execute 249 sec latency 38.342 ms 2024-09-21T04:15:31.012 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:15:30 smithi089 ceph-mon[64076]: pgmap v47: 97 pgs: 97 active+clean; 328 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 6.7 MiB/s wr, 688 op/s 2024-09-21T04:15:31.347 INFO:teuthology.orchestra.run.smithi089.stdout: 5 162642 10.55 MB/sec execute 250 sec latency 55.092 ms 2024-09-21T04:15:31.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:15:30 smithi195 ceph-mon[49435]: pgmap v47: 97 pgs: 97 active+clean; 328 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 6.7 MiB/s wr, 688 op/s 2024-09-21T04:15:32.353 INFO:teuthology.orchestra.run.smithi089.stdout: 5 162982 10.56 MB/sec execute 251 sec latency 64.815 ms 2024-09-21T04:15:32.578 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:15:32 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:15:32.578 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:15:32 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:15:32.578 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:15:32 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T04:15:32.578 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:15:32 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T04:15:32.578 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:15:32 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:15:32.578 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:15:32 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:15:32.578 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:15:32 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T04:15:32.579 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:15:32 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "versions"} : dispatch 2024-09-21T04:15:32.579 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:15:32 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "versions"} : dispatch 2024-09-21T04:15:32.579 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:15:32 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "versions"} : dispatch 2024-09-21T04:15:32.579 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:15:32 smithi089 ceph-mon[64076]: Upgrade: Setting container_image for all crash 2024-09-21T04:15:32.579 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:15:32 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:15:32.579 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:15:32 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi089"} : dispatch 2024-09-21T04:15:32.579 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:15:32 smithi089 ceph-mon[64076]: pgmap v48: 97 pgs: 97 active+clean; 328 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.1 MiB/s wr, 389 op/s 2024-09-21T04:15:32.579 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:15:32 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi089"}]': finished 2024-09-21T04:15:32.579 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:15:32 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi195"} : dispatch 2024-09-21T04:15:32.579 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:15:32 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi195"}]': finished 2024-09-21T04:15:32.579 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:15:32 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16} : dispatch 2024-09-21T04:15:32.579 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:15:32 smithi089 ceph-mon[64076]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-09-21T04:15:32.579 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:15:32 smithi089 ceph-mon[64076]: Upgrade: osd.0 is safe to restart 2024-09-21T04:15:32.579 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:15:32 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T04:15:32.859 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:15:32 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:15:32.859 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:15:32 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:15:32.860 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:15:32 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T04:15:32.860 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:15:32 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T04:15:32.860 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:15:32 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:15:32.860 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:15:32 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:15:32.860 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:15:32 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T04:15:32.860 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:15:32 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "versions"} : dispatch 2024-09-21T04:15:32.860 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:15:32 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "versions"} : dispatch 2024-09-21T04:15:32.860 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:15:32 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "versions"} : dispatch 2024-09-21T04:15:32.860 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:15:32 smithi195 ceph-mon[49435]: Upgrade: Setting container_image for all crash 2024-09-21T04:15:32.860 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:15:32 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:15:32.860 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:15:32 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi089"} : dispatch 2024-09-21T04:15:32.860 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:15:32 smithi195 ceph-mon[49435]: pgmap v48: 97 pgs: 97 active+clean; 328 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.1 MiB/s wr, 389 op/s 2024-09-21T04:15:32.860 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:15:32 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi089"}]': finished 2024-09-21T04:15:32.860 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:15:32 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi195"} : dispatch 2024-09-21T04:15:32.861 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:15:32 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi195"}]': finished 2024-09-21T04:15:32.861 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:15:32 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16} : dispatch 2024-09-21T04:15:32.861 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:15:32 smithi195 ceph-mon[49435]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-09-21T04:15:32.861 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:15:32 smithi195 ceph-mon[49435]: Upgrade: osd.0 is safe to restart 2024-09-21T04:15:32.861 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:15:32 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T04:15:33.353 INFO:teuthology.orchestra.run.smithi089.stdout: 5 163278 10.58 MB/sec execute 252 sec latency 73.400 ms 2024-09-21T04:15:34.353 INFO:teuthology.orchestra.run.smithi089.stdout: 5 163576 10.56 MB/sec execute 253 sec latency 63.962 ms 2024-09-21T04:15:34.934 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:15:34 smithi089 ceph-mon[64076]: Upgrade: Updating osd.0 2024-09-21T04:15:34.935 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:15:34 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:15:34.935 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:15:34 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "auth get", "entity": "osd.0"} : dispatch 2024-09-21T04:15:34.935 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:15:34 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T04:15:34.935 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:15:34 smithi089 ceph-mon[64076]: Deploying daemon osd.0 on smithi089 2024-09-21T04:15:34.935 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:15:34 smithi089 ceph-mon[64076]: pgmap v49: 97 pgs: 97 active+clean; 337 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 8.6 KiB/s rd, 6.7 MiB/s wr, 613 op/s 2024-09-21T04:15:35.109 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:15:34 smithi195 ceph-mon[49435]: Upgrade: Updating osd.0 2024-09-21T04:15:35.109 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:15:34 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:15:35.109 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:15:34 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "auth get", "entity": "osd.0"} : dispatch 2024-09-21T04:15:35.109 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:15:34 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T04:15:35.109 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:15:34 smithi195 ceph-mon[49435]: Deploying daemon osd.0 on smithi089 2024-09-21T04:15:35.109 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:15:34 smithi195 ceph-mon[49435]: pgmap v49: 97 pgs: 97 active+clean; 337 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 8.6 KiB/s rd, 6.7 MiB/s wr, 613 op/s 2024-09-21T04:15:35.353 INFO:teuthology.orchestra.run.smithi089.stdout: 5 163894 10.53 MB/sec execute 254 sec latency 71.508 ms 2024-09-21T04:15:35.967 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:15:35 smithi089 ceph-mon[64076]: mgrmap e44: smithi089.srlyxf(active, since 92s), standbys: smithi195.cgotzm 2024-09-21T04:15:36.109 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:15:35 smithi195 ceph-mon[49435]: mgrmap e44: smithi089.srlyxf(active, since 92s), standbys: smithi195.cgotzm 2024-09-21T04:15:36.353 INFO:teuthology.orchestra.run.smithi089.stdout: 5 164183 10.50 MB/sec execute 255 sec latency 60.311 ms 2024-09-21T04:15:36.991 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:15:36 smithi089 ceph-mon[64076]: pgmap v50: 97 pgs: 97 active+clean; 337 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.1 MiB/s wr, 401 op/s 2024-09-21T04:15:37.353 INFO:teuthology.orchestra.run.smithi089.stdout: 5 164468 10.50 MB/sec execute 256 sec latency 49.758 ms 2024-09-21T04:15:37.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:15:36 smithi195 ceph-mon[49435]: pgmap v50: 97 pgs: 97 active+clean; 337 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.1 MiB/s wr, 401 op/s 2024-09-21T04:15:38.353 INFO:teuthology.orchestra.run.smithi089.stdout: 5 164894 10.49 MB/sec execute 257 sec latency 49.432 ms 2024-09-21T04:15:38.794 INFO:journalctl@ceph.osd.0.smithi089.stdout:Sep 21 04:15:38 smithi089 systemd[1]: Stopping Ceph osd.0 for 1b9b79fc-77ce-11ef-baf5-efdc52797490... 2024-09-21T04:15:39.048 INFO:journalctl@ceph.osd.0.smithi089.stdout:Sep 21 04:15:38 smithi089 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-0[33916]: 2024-09-21T04:15:38.990+0000 7f5357d6f700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-21T04:15:39.048 INFO:journalctl@ceph.osd.0.smithi089.stdout:Sep 21 04:15:38 smithi089 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-0[33916]: 2024-09-21T04:15:38.990+0000 7f5357d6f700 -1 osd.0 74 *** Got signal Terminated *** 2024-09-21T04:15:39.048 INFO:journalctl@ceph.osd.0.smithi089.stdout:Sep 21 04:15:38 smithi089 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-0[33916]: 2024-09-21T04:15:38.990+0000 7f5357d6f700 -1 osd.0 74 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-21T04:15:39.049 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:15:38 smithi089 ceph-mon[64076]: pgmap v51: 97 pgs: 97 active+clean; 333 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.7 MiB/s wr, 498 op/s 2024-09-21T04:15:39.353 INFO:teuthology.orchestra.run.smithi089.stdout: 5 165259 10.47 MB/sec execute 258 sec latency 59.299 ms 2024-09-21T04:15:39.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:15:38 smithi195 ceph-mon[49435]: pgmap v51: 97 pgs: 97 active+clean; 333 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.7 MiB/s wr, 498 op/s 2024-09-21T04:15:40.079 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:15:39 smithi089 ceph-mon[64076]: osd.0 marked itself down and dead 2024-09-21T04:15:40.354 INFO:teuthology.orchestra.run.smithi089.stdout: 5 165527 10.45 MB/sec execute 259 sec latency 343.681 ms 2024-09-21T04:15:40.358 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:15:39 smithi195 ceph-mon[49435]: osd.0 marked itself down and dead 2024-09-21T04:15:40.955 INFO:journalctl@ceph.osd.0.smithi089.stdout:Sep 21 04:15:40 smithi089 podman[68388]: 2024-09-21 04:15:40.637618703 +0000 UTC m=+1.862906201 container died db7c0e5020e741a5752ec77ded5bbd6a3802b132f1c8982eb04766b5cfc2bd11 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-0, name=centos-stream, vcs-type=git, io.openshift.expose-services=, vendor=Red Hat, Inc., com.redhat.component=centos-stream-container, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, CEPH_POINT_RELEASE=-17.2.0, distribution-scope=public, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.k8s.display-name=CentOS Stream 8, GIT_REPO=https://github.com/ceph/ceph-container.git, io.buildah.version=1.19.8, RELEASE=HEAD, version=8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, ceph=True, architecture=x86_64, GIT_CLEAN=True, io.openshift.tags=base centos centos-stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., build-date=2022-05-03T08:36:31.336870, release=754, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream) 2024-09-21T04:15:41.207 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:15:40 smithi089 ceph-mon[64076]: pgmap v52: 97 pgs: 97 active+clean; 321 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 5.4 MiB/s wr, 530 op/s 2024-09-21T04:15:41.207 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:15:40 smithi089 ceph-mon[64076]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-21T04:15:41.207 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:15:40 smithi089 ceph-mon[64076]: osdmap e75: 8 total, 7 up, 8 in 2024-09-21T04:15:41.207 INFO:journalctl@ceph.osd.0.smithi089.stdout:Sep 21 04:15:41 smithi089 podman[68388]: 2024-09-21 04:15:41.06022222 +0000 UTC m=+2.285509708 container cleanup db7c0e5020e741a5752ec77ded5bbd6a3802b132f1c8982eb04766b5cfc2bd11 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-0, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.component=centos-stream-container, vendor=Red Hat, Inc., io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., CEPH_POINT_RELEASE=-17.2.0, build-date=2022-05-03T08:36:31.336870, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, RELEASE=HEAD, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.k8s.display-name=CentOS Stream 8, io.buildah.version=1.19.8, maintainer=Guillaume Abrioux , io.openshift.tags=base centos centos-stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, architecture=x86_64, name=centos-stream, version=8, io.openshift.expose-services=, vcs-type=git, GIT_CLEAN=True, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, distribution-scope=public, GIT_BRANCH=HEAD, ceph=True, release=754) 2024-09-21T04:15:41.207 INFO:journalctl@ceph.osd.0.smithi089.stdout:Sep 21 04:15:41 smithi089 bash[68388]: ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-0 2024-09-21T04:15:41.353 INFO:teuthology.orchestra.run.smithi089.stdout: 5 165606 10.41 MB/sec execute 260 sec latency 1020.978 ms 2024-09-21T04:15:41.358 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:15:40 smithi195 ceph-mon[49435]: pgmap v52: 97 pgs: 97 active+clean; 321 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 5.4 MiB/s wr, 530 op/s 2024-09-21T04:15:41.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:15:40 smithi195 ceph-mon[49435]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-21T04:15:41.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:15:40 smithi195 ceph-mon[49435]: osdmap e75: 8 total, 7 up, 8 in 2024-09-21T04:15:41.718 INFO:journalctl@ceph.osd.0.smithi089.stdout:Sep 21 04:15:41 smithi089 podman[68405]: 2024-09-21 04:15:41.642916285 +0000 UTC m=+0.989359196 container remove db7c0e5020e741a5752ec77ded5bbd6a3802b132f1c8982eb04766b5cfc2bd11 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-0, GIT_CLEAN=True, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, build-date=2022-05-03T08:36:31.336870, io.openshift.expose-services=, io.openshift.tags=base centos centos-stream, name=centos-stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, com.redhat.component=centos-stream-container, ceph=True, CEPH_POINT_RELEASE=-17.2.0, maintainer=Guillaume Abrioux , release=754, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vcs-type=git, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_BRANCH=HEAD, distribution-scope=public, version=8, io.buildah.version=1.19.8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_REPO=https://github.com/ceph/ceph-container.git, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., architecture=x86_64, vendor=Red Hat, Inc., vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, RELEASE=HEAD, io.k8s.display-name=CentOS Stream 8) 2024-09-21T04:15:42.232 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:15:42 smithi089 ceph-mon[64076]: osdmap e76: 8 total, 7 up, 8 in 2024-09-21T04:15:42.232 INFO:journalctl@ceph.osd.0.smithi089.stdout:Sep 21 04:15:42 smithi089 podman[68489]: 2024-09-21 04:15:42.045593404 +0000 UTC m=+0.017264558 image pull 48ed0a2cf89d6d0371a2c6185a5869a747265f6abd5ac2a74f00cc32d5628721 quay.ceph.io/ceph-ci/ceph@sha256:2195e5f178be0a9390728e2a14088236d8fe6f82635f5c84418fe37a04457e00 2024-09-21T04:15:42.232 INFO:journalctl@ceph.osd.0.smithi089.stdout:Sep 21 04:15:42 smithi089 podman[68489]: 2024-09-21 04:15:42.16466703 +0000 UTC m=+0.136338173 container create 6a1209538df71b7732688a403afba1ad37467114015fb92c46e76403075b4885 (image=quay.ceph.io/ceph-ci/ceph@sha256:2195e5f178be0a9390728e2a14088236d8fe6f82635f5c84418fe37a04457e00, name=ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-0-deactivate, ceph=True, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , RELEASE=wip-guits-main-2024-09-20-2310-f36283e, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, CEPH_POINT_RELEASE=) 2024-09-21T04:15:42.354 INFO:teuthology.orchestra.run.smithi089.stdout: 5 165893 10.42 MB/sec execute 261 sec latency 47.715 ms 2024-09-21T04:15:42.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:15:42 smithi195 ceph-mon[49435]: osdmap e76: 8 total, 7 up, 8 in 2024-09-21T04:15:42.488 INFO:journalctl@ceph.osd.0.smithi089.stdout:Sep 21 04:15:42 smithi089 podman[68489]: 2024-09-21 04:15:42.467755549 +0000 UTC m=+0.439426693 container init 6a1209538df71b7732688a403afba1ad37467114015fb92c46e76403075b4885 (image=quay.ceph.io/ceph-ci/ceph@sha256:2195e5f178be0a9390728e2a14088236d8fe6f82635f5c84418fe37a04457e00, name=ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-0-deactivate, CEPH_POINT_RELEASE=, RELEASE=wip-guits-main-2024-09-20-2310-f36283e, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, ceph=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS) 2024-09-21T04:15:42.488 INFO:journalctl@ceph.osd.0.smithi089.stdout:Sep 21 04:15:42 smithi089 podman[68489]: 2024-09-21 04:15:42.473116414 +0000 UTC m=+0.444787561 container start 6a1209538df71b7732688a403afba1ad37467114015fb92c46e76403075b4885 (image=quay.ceph.io/ceph-ci/ceph@sha256:2195e5f178be0a9390728e2a14088236d8fe6f82635f5c84418fe37a04457e00, name=ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-0-deactivate, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , RELEASE=wip-guits-main-2024-09-20-2310-f36283e, GIT_CLEAN=True, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731) 2024-09-21T04:15:42.746 INFO:journalctl@ceph.osd.0.smithi089.stdout:Sep 21 04:15:42 smithi089 podman[68489]: 2024-09-21 04:15:42.635307107 +0000 UTC m=+0.606978260 container attach 6a1209538df71b7732688a403afba1ad37467114015fb92c46e76403075b4885 (image=quay.ceph.io/ceph-ci/ceph@sha256:2195e5f178be0a9390728e2a14088236d8fe6f82635f5c84418fe37a04457e00, name=ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-0-deactivate, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, RELEASE=wip-guits-main-2024-09-20-2310-f36283e, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD) 2024-09-21T04:15:42.998 INFO:journalctl@ceph.osd.0.smithi089.stdout:Sep 21 04:15:42 smithi089 podman[68489]: 2024-09-21 04:15:42.801158797 +0000 UTC m=+0.772829948 container died 6a1209538df71b7732688a403afba1ad37467114015fb92c46e76403075b4885 (image=quay.ceph.io/ceph-ci/ceph@sha256:2195e5f178be0a9390728e2a14088236d8fe6f82635f5c84418fe37a04457e00, name=ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-0-deactivate, RELEASE=wip-guits-main-2024-09-20-2310-f36283e, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, ceph=True, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731) 2024-09-21T04:15:43.249 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:15:43 smithi089 ceph-mon[64076]: pgmap v55: 97 pgs: 13 stale+active+clean, 84 active+clean; 321 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 6.5 KiB/s rd, 2.6 MiB/s wr, 319 op/s 2024-09-21T04:15:43.354 INFO:teuthology.orchestra.run.smithi089.stdout: 5 166380 10.47 MB/sec execute 262 sec latency 27.622 ms 2024-09-21T04:15:43.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:15:43 smithi195 ceph-mon[49435]: pgmap v55: 97 pgs: 13 stale+active+clean, 84 active+clean; 321 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 6.5 KiB/s rd, 2.6 MiB/s wr, 319 op/s 2024-09-21T04:15:44.021 INFO:journalctl@ceph.osd.0.smithi089.stdout:Sep 21 04:15:43 smithi089 podman[68511]: 2024-09-21 04:15:43.844468606 +0000 UTC m=+1.036627386 container remove 6a1209538df71b7732688a403afba1ad37467114015fb92c46e76403075b4885 (image=quay.ceph.io/ceph-ci/ceph@sha256:2195e5f178be0a9390728e2a14088236d8fe6f82635f5c84418fe37a04457e00, name=ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-0-deactivate, ceph=True, org.label-schema.schema-version=1.0, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, io.buildah.version=1.37.2, RELEASE=wip-guits-main-2024-09-20-2310-f36283e, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS) 2024-09-21T04:15:44.021 INFO:journalctl@ceph.osd.0.smithi089.stdout:Sep 21 04:15:43 smithi089 systemd[1]: ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490@osd.0.service: Deactivated successfully. 2024-09-21T04:15:44.021 INFO:journalctl@ceph.osd.0.smithi089.stdout:Sep 21 04:15:43 smithi089 systemd[1]: Stopped Ceph osd.0 for 1b9b79fc-77ce-11ef-baf5-efdc52797490. 2024-09-21T04:15:44.021 INFO:journalctl@ceph.osd.0.smithi089.stdout:Sep 21 04:15:43 smithi089 systemd[1]: ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490@osd.0.service: Consumed 2min 43.358s CPU time. 2024-09-21T04:15:44.285 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:15:44 smithi089 ceph-mon[64076]: Health check failed: Degraded data redundancy: 203/1593 objects degraded (12.743%), 29 pgs degraded (PG_DEGRADED) 2024-09-21T04:15:44.285 INFO:journalctl@ceph.osd.0.smithi089.stdout:Sep 21 04:15:44 smithi089 systemd[1]: Starting Ceph osd.0 for 1b9b79fc-77ce-11ef-baf5-efdc52797490... 2024-09-21T04:15:44.354 INFO:teuthology.orchestra.run.smithi089.stdout: 5 166749 10.49 MB/sec execute 263 sec latency 40.468 ms 2024-09-21T04:15:44.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:15:44 smithi195 ceph-mon[49435]: Health check failed: Degraded data redundancy: 203/1593 objects degraded (12.743%), 29 pgs degraded (PG_DEGRADED) 2024-09-21T04:15:44.538 INFO:journalctl@ceph.osd.0.smithi089.stdout:Sep 21 04:15:44 smithi089 podman[68606]: 2024-09-21 04:15:44.396521003 +0000 UTC m=+0.021139631 image pull 48ed0a2cf89d6d0371a2c6185a5869a747265f6abd5ac2a74f00cc32d5628721 quay.ceph.io/ceph-ci/ceph@sha256:2195e5f178be0a9390728e2a14088236d8fe6f82635f5c84418fe37a04457e00 2024-09-21T04:15:44.538 INFO:journalctl@ceph.osd.0.smithi089.stdout:Sep 21 04:15:44 smithi089 podman[68606]: 2024-09-21 04:15:44.529683707 +0000 UTC m=+0.154302328 container create 7773d089ec9eca32aed54ebe58ab79fda1773283f1757b0f25372fc30e1b8e0d (image=quay.ceph.io/ceph-ci/ceph@sha256:2195e5f178be0a9390728e2a14088236d8fe6f82635f5c84418fe37a04457e00, name=ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-0-activate, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, RELEASE=wip-guits-main-2024-09-20-2310-f36283e, GIT_CLEAN=True, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, CEPH_POINT_RELEASE=) 2024-09-21T04:15:45.040 INFO:journalctl@ceph.osd.0.smithi089.stdout:Sep 21 04:15:44 smithi089 podman[68606]: 2024-09-21 04:15:44.950459311 +0000 UTC m=+0.575077927 container init 7773d089ec9eca32aed54ebe58ab79fda1773283f1757b0f25372fc30e1b8e0d (image=quay.ceph.io/ceph-ci/ceph@sha256:2195e5f178be0a9390728e2a14088236d8fe6f82635f5c84418fe37a04457e00, name=ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-0-activate, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, io.buildah.version=1.37.2, GIT_CLEAN=True, ceph=True, org.label-schema.build-date=20240731, RELEASE=wip-guits-main-2024-09-20-2310-f36283e, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0) 2024-09-21T04:15:45.040 INFO:journalctl@ceph.osd.0.smithi089.stdout:Sep 21 04:15:44 smithi089 podman[68606]: 2024-09-21 04:15:44.967900288 +0000 UTC m=+0.592518895 container start 7773d089ec9eca32aed54ebe58ab79fda1773283f1757b0f25372fc30e1b8e0d (image=quay.ceph.io/ceph-ci/ceph@sha256:2195e5f178be0a9390728e2a14088236d8fe6f82635f5c84418fe37a04457e00, name=ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-0-activate, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-guits-main-2024-09-20-2310-f36283e, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731) 2024-09-21T04:15:45.292 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:15:45 smithi089 ceph-mon[64076]: pgmap v56: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 329 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 8.4 KiB/s rd, 5.3 MiB/s wr, 571 op/s; 203/1593 objects degraded (12.743%) 2024-09-21T04:15:45.293 INFO:journalctl@ceph.osd.0.smithi089.stdout:Sep 21 04:15:45 smithi089 podman[68606]: 2024-09-21 04:15:45.067176997 +0000 UTC m=+0.691795621 container attach 7773d089ec9eca32aed54ebe58ab79fda1773283f1757b0f25372fc30e1b8e0d (image=quay.ceph.io/ceph-ci/ceph@sha256:2195e5f178be0a9390728e2a14088236d8fe6f82635f5c84418fe37a04457e00, name=ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-0-activate, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , RELEASE=wip-guits-main-2024-09-20-2310-f36283e, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.license=GPLv2) 2024-09-21T04:15:45.293 INFO:journalctl@ceph.osd.0.smithi089.stdout:Sep 21 04:15:45 smithi089 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-0-activate[68620]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T04:15:45.293 INFO:journalctl@ceph.osd.0.smithi089.stdout:Sep 21 04:15:45 smithi089 bash[68606]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T04:15:45.293 INFO:journalctl@ceph.osd.0.smithi089.stdout:Sep 21 04:15:45 smithi089 bash[68606]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T04:15:45.293 INFO:journalctl@ceph.osd.0.smithi089.stdout:Sep 21 04:15:45 smithi089 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-0-activate[68620]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T04:15:45.354 INFO:teuthology.orchestra.run.smithi089.stdout: 5 167004 10.48 MB/sec execute 264 sec latency 52.805 ms 2024-09-21T04:15:45.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:15:45 smithi195 ceph-mon[49435]: pgmap v56: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 329 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 8.4 KiB/s rd, 5.3 MiB/s wr, 571 op/s; 203/1593 objects degraded (12.743%) 2024-09-21T04:15:46.354 INFO:teuthology.orchestra.run.smithi089.stdout: 5 167259 10.45 MB/sec execute 265 sec latency 39.263 ms 2024-09-21T04:15:47.071 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:15:47 smithi089 ceph-mon[64076]: pgmap v57: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 329 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 8.4 KiB/s rd, 4.4 MiB/s wr, 426 op/s; 203/1593 objects degraded (12.743%) 2024-09-21T04:15:47.354 INFO:teuthology.orchestra.run.smithi089.stdout: 5 167555 10.42 MB/sec execute 266 sec latency 43.346 ms 2024-09-21T04:15:47.358 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:15:47 smithi195 ceph-mon[49435]: pgmap v57: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 329 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 8.4 KiB/s rd, 4.4 MiB/s wr, 426 op/s; 203/1593 objects degraded (12.743%) 2024-09-21T04:15:48.354 INFO:teuthology.orchestra.run.smithi089.stdout: 5 167837 10.40 MB/sec execute 267 sec latency 43.683 ms 2024-09-21T04:15:48.530 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:15:48 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:15:48.530 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:15:48 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T04:15:48.530 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:15:48 smithi089 ceph-mon[64076]: pgmap v58: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 326 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 1.9 KiB/s rd, 2.8 MiB/s wr, 256 op/s; 205/1599 objects degraded (12.821%) 2024-09-21T04:15:48.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:15:48 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:15:48.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:15:48 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T04:15:48.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:15:48 smithi195 ceph-mon[49435]: pgmap v58: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 326 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 1.9 KiB/s rd, 2.8 MiB/s wr, 256 op/s; 205/1599 objects degraded (12.821%) 2024-09-21T04:15:49.354 INFO:teuthology.orchestra.run.smithi089.stdout: 5 168105 10.39 MB/sec execute 268 sec latency 54.987 ms 2024-09-21T04:15:49.805 INFO:journalctl@ceph.osd.0.smithi089.stdout:Sep 21 04:15:49 smithi089 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-0-activate[68620]: --> Failed to activate via raw: did not find any matching OSD to activate 2024-09-21T04:15:49.805 INFO:journalctl@ceph.osd.0.smithi089.stdout:Sep 21 04:15:49 smithi089 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-0-activate[68620]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T04:15:49.805 INFO:journalctl@ceph.osd.0.smithi089.stdout:Sep 21 04:15:49 smithi089 bash[68606]: --> Failed to activate via raw: did not find any matching OSD to activate 2024-09-21T04:15:49.805 INFO:journalctl@ceph.osd.0.smithi089.stdout:Sep 21 04:15:49 smithi089 bash[68606]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T04:15:49.805 INFO:journalctl@ceph.osd.0.smithi089.stdout:Sep 21 04:15:49 smithi089 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-0-activate[68620]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T04:15:49.805 INFO:journalctl@ceph.osd.0.smithi089.stdout:Sep 21 04:15:49 smithi089 bash[68606]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T04:15:50.064 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:15:49 smithi089 ceph-mon[64076]: Health check update: Degraded data redundancy: 205/1599 objects degraded (12.821%), 29 pgs degraded (PG_DEGRADED) 2024-09-21T04:15:50.064 INFO:journalctl@ceph.osd.0.smithi089.stdout:Sep 21 04:15:49 smithi089 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-0-activate[68620]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-09-21T04:15:50.064 INFO:journalctl@ceph.osd.0.smithi089.stdout:Sep 21 04:15:49 smithi089 bash[68606]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-09-21T04:15:50.064 INFO:journalctl@ceph.osd.0.smithi089.stdout:Sep 21 04:15:49 smithi089 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-0-activate[68620]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_4 --path /var/lib/ceph/osd/ceph-0 --no-mon-config 2024-09-21T04:15:50.064 INFO:journalctl@ceph.osd.0.smithi089.stdout:Sep 21 04:15:49 smithi089 bash[68606]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_4 --path /var/lib/ceph/osd/ceph-0 --no-mon-config 2024-09-21T04:15:50.354 INFO:teuthology.orchestra.run.smithi089.stdout: 5 168583 10.39 MB/sec execute 269 sec latency 36.411 ms 2024-09-21T04:15:50.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:15:49 smithi195 ceph-mon[49435]: Health check update: Degraded data redundancy: 205/1599 objects degraded (12.821%), 29 pgs degraded (PG_DEGRADED) 2024-09-21T04:15:50.573 INFO:journalctl@ceph.osd.0.smithi089.stdout:Sep 21 04:15:50 smithi089 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-0-activate[68620]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_4 /var/lib/ceph/osd/ceph-0/block 2024-09-21T04:15:50.573 INFO:journalctl@ceph.osd.0.smithi089.stdout:Sep 21 04:15:50 smithi089 bash[68606]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_4 /var/lib/ceph/osd/ceph-0/block 2024-09-21T04:15:50.573 INFO:journalctl@ceph.osd.0.smithi089.stdout:Sep 21 04:15:50 smithi089 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-0-activate[68620]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-0/block 2024-09-21T04:15:50.573 INFO:journalctl@ceph.osd.0.smithi089.stdout:Sep 21 04:15:50 smithi089 bash[68606]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-0/block 2024-09-21T04:15:50.573 INFO:journalctl@ceph.osd.0.smithi089.stdout:Sep 21 04:15:50 smithi089 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-0-activate[68620]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-09-21T04:15:50.573 INFO:journalctl@ceph.osd.0.smithi089.stdout:Sep 21 04:15:50 smithi089 bash[68606]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-09-21T04:15:50.573 INFO:journalctl@ceph.osd.0.smithi089.stdout:Sep 21 04:15:50 smithi089 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-0-activate[68620]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-09-21T04:15:50.573 INFO:journalctl@ceph.osd.0.smithi089.stdout:Sep 21 04:15:50 smithi089 bash[68606]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-09-21T04:15:50.574 INFO:journalctl@ceph.osd.0.smithi089.stdout:Sep 21 04:15:50 smithi089 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-0-activate[68620]: --> ceph-volume lvm activate successful for osd ID: 0 2024-09-21T04:15:50.574 INFO:journalctl@ceph.osd.0.smithi089.stdout:Sep 21 04:15:50 smithi089 bash[68606]: --> ceph-volume lvm activate successful for osd ID: 0 2024-09-21T04:15:50.574 INFO:journalctl@ceph.osd.0.smithi089.stdout:Sep 21 04:15:50 smithi089 podman[68606]: 2024-09-21 04:15:50.362439882 +0000 UTC m=+5.987058487 container died 7773d089ec9eca32aed54ebe58ab79fda1773283f1757b0f25372fc30e1b8e0d (image=quay.ceph.io/ceph-ci/ceph@sha256:2195e5f178be0a9390728e2a14088236d8fe6f82635f5c84418fe37a04457e00, name=ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-0-activate, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, RELEASE=wip-guits-main-2024-09-20-2310-f36283e, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23) 2024-09-21T04:15:51.074 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:15:50 smithi089 ceph-mon[64076]: pgmap v59: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 332 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 5.2 KiB/s rd, 4.7 MiB/s wr, 438 op/s; 223/1692 objects degraded (13.180%) 2024-09-21T04:15:51.354 INFO:teuthology.orchestra.run.smithi089.stdout: 5 168988 10.37 MB/sec execute 270 sec latency 47.516 ms 2024-09-21T04:15:51.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:15:50 smithi195 ceph-mon[49435]: pgmap v59: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 332 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 5.2 KiB/s rd, 4.7 MiB/s wr, 438 op/s; 223/1692 objects degraded (13.180%) 2024-09-21T04:15:51.362 INFO:journalctl@ceph.osd.0.smithi089.stdout:Sep 21 04:15:51 smithi089 podman[68840]: 2024-09-21 04:15:51.224261046 +0000 UTC m=+0.855277705 container remove 7773d089ec9eca32aed54ebe58ab79fda1773283f1757b0f25372fc30e1b8e0d (image=quay.ceph.io/ceph-ci/ceph@sha256:2195e5f178be0a9390728e2a14088236d8fe6f82635f5c84418fe37a04457e00, name=ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-0-activate, RELEASE=wip-guits-main-2024-09-20-2310-f36283e, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, ceph=True, CEPH_POINT_RELEASE=, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-21T04:15:51.612 INFO:journalctl@ceph.osd.0.smithi089.stdout:Sep 21 04:15:51 smithi089 podman[68895]: 2024-09-21 04:15:51.378306596 +0000 UTC m=+0.020974662 image pull 48ed0a2cf89d6d0371a2c6185a5869a747265f6abd5ac2a74f00cc32d5628721 quay.ceph.io/ceph-ci/ceph@sha256:2195e5f178be0a9390728e2a14088236d8fe6f82635f5c84418fe37a04457e00 2024-09-21T04:15:51.612 INFO:journalctl@ceph.osd.0.smithi089.stdout:Sep 21 04:15:51 smithi089 podman[68895]: 2024-09-21 04:15:51.51045442 +0000 UTC m=+0.153122462 container create 3df0adea59fbe887dc16f6fe4727ab17845fbb1a2e133b9e8cbeae51f91f0160 (image=quay.ceph.io/ceph-ci/ceph@sha256:2195e5f178be0a9390728e2a14088236d8fe6f82635f5c84418fe37a04457e00, name=ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-0, org.label-schema.schema-version=1.0, RELEASE=wip-guits-main-2024-09-20-2310-f36283e, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-21T04:15:52.114 INFO:journalctl@ceph.osd.0.smithi089.stdout:Sep 21 04:15:51 smithi089 podman[68895]: 2024-09-21 04:15:51.938759838 +0000 UTC m=+0.581427898 container init 3df0adea59fbe887dc16f6fe4727ab17845fbb1a2e133b9e8cbeae51f91f0160 (image=quay.ceph.io/ceph-ci/ceph@sha256:2195e5f178be0a9390728e2a14088236d8fe6f82635f5c84418fe37a04457e00, name=ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-0, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, RELEASE=wip-guits-main-2024-09-20-2310-f36283e) 2024-09-21T04:15:52.114 INFO:journalctl@ceph.osd.0.smithi089.stdout:Sep 21 04:15:51 smithi089 podman[68895]: 2024-09-21 04:15:51.943536481 +0000 UTC m=+0.586204522 container start 3df0adea59fbe887dc16f6fe4727ab17845fbb1a2e133b9e8cbeae51f91f0160 (image=quay.ceph.io/ceph-ci/ceph@sha256:2195e5f178be0a9390728e2a14088236d8fe6f82635f5c84418fe37a04457e00, name=ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-0, maintainer=Guillaume Abrioux , ceph=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_CLEAN=True, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, RELEASE=wip-guits-main-2024-09-20-2310-f36283e) 2024-09-21T04:15:52.114 INFO:journalctl@ceph.osd.0.smithi089.stdout:Sep 21 04:15:51 smithi089 ceph-osd[68913]: -- 172.21.15.89:0/4246557306 <== mon.1 v2:172.21.15.195:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55a0448821e0 con 0x55a043a87c00 2024-09-21T04:15:52.354 INFO:teuthology.orchestra.run.smithi089.stdout: 5 169340 10.38 MB/sec execute 271 sec latency 57.829 ms 2024-09-21T04:15:52.374 INFO:journalctl@ceph.osd.0.smithi089.stdout:Sep 21 04:15:52 smithi089 bash[68895]: 3df0adea59fbe887dc16f6fe4727ab17845fbb1a2e133b9e8cbeae51f91f0160 2024-09-21T04:15:52.374 INFO:journalctl@ceph.osd.0.smithi089.stdout:Sep 21 04:15:52 smithi089 systemd[1]: Started Ceph osd.0 for 1b9b79fc-77ce-11ef-baf5-efdc52797490. 2024-09-21T04:15:53.142 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:15:52 smithi089 ceph-mon[64076]: pgmap v60: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 332 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 4.8 KiB/s rd, 4.2 MiB/s wr, 398 op/s; 223/1692 objects degraded (13.180%) 2024-09-21T04:15:53.142 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:15:52 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:15:53.142 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:15:52 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:15:53.142 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:15:52 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T04:15:53.355 INFO:teuthology.orchestra.run.smithi089.stdout: 5 169653 10.39 MB/sec execute 272 sec latency 64.685 ms 2024-09-21T04:15:53.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:15:52 smithi195 ceph-mon[49435]: pgmap v60: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 332 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 4.8 KiB/s rd, 4.2 MiB/s wr, 398 op/s; 223/1692 objects degraded (13.180%) 2024-09-21T04:15:53.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:15:52 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:15:53.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:15:52 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:15:53.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:15:52 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T04:15:54.355 INFO:teuthology.orchestra.run.smithi089.stdout: 5 169979 10.40 MB/sec execute 273 sec latency 46.295 ms 2024-09-21T04:15:54.410 INFO:journalctl@ceph.osd.0.smithi089.stdout:Sep 21 04:15:54 smithi089 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-0[68909]: 2024-09-21T04:15:54.255+0000 7f4f122fe640 -1 Falling back to public interface 2024-09-21T04:15:54.619 INFO:teuthology.orchestra.run.smithi089.stdout:true 2024-09-21T04:15:55.189 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:15:54 smithi089 ceph-mon[64076]: pgmap v61: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 323 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 7.7 KiB/s rd, 5.7 MiB/s wr, 579 op/s; 219/1593 objects degraded (13.748%) 2024-09-21T04:15:55.190 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:15:54 smithi089 ceph-mon[64076]: Health check update: Degraded data redundancy: 219/1593 objects degraded (13.748%), 29 pgs degraded (PG_DEGRADED) 2024-09-21T04:15:55.355 INFO:teuthology.orchestra.run.smithi089.stdout: 5 170219 10.39 MB/sec execute 274 sec latency 58.910 ms 2024-09-21T04:15:55.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:15:54 smithi195 ceph-mon[49435]: pgmap v61: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 323 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 7.7 KiB/s rd, 5.7 MiB/s wr, 579 op/s; 219/1593 objects degraded (13.748%) 2024-09-21T04:15:55.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:15:54 smithi195 ceph-mon[49435]: Health check update: Degraded data redundancy: 219/1593 objects degraded (13.748%), 29 pgs degraded (PG_DEGRADED) 2024-09-21T04:15:55.629 INFO:teuthology.orchestra.run.smithi089.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-21T04:15:55.630 INFO:teuthology.orchestra.run.smithi089.stdout:alertmanager.smithi089 smithi089 *:9093,9094 running (4m) 30s ago 13m 19.5M - 0.27.0 11f11916f8cd 65dfcbcafee1 2024-09-21T04:15:55.630 INFO:teuthology.orchestra.run.smithi089.stdout:crash.smithi089 smithi089 running (46s) 30s ago 13m 6891k - 19.3.0-5103-gf36283e7 48ed0a2cf89d 6c415a5e173e 2024-09-21T04:15:55.630 INFO:teuthology.orchestra.run.smithi089.stdout:crash.smithi195 smithi195 running (39s) 33s ago 12m 6891k - 19.3.0-5103-gf36283e7 48ed0a2cf89d 78ce05a4e9ab 2024-09-21T04:15:55.630 INFO:teuthology.orchestra.run.smithi089.stdout:grafana.smithi089 smithi089 *:3000 running (3m) 30s ago 12m 45.3M - dad864ee21e9 ec4d6738bce7 2024-09-21T04:15:55.630 INFO:teuthology.orchestra.run.smithi089.stdout:mds.foofs.smithi089.osufws smithi089 running (7m) 30s ago 7m 38.2M - 17.2.0 e1d6a67b021e 664d41859bea 2024-09-21T04:15:55.630 INFO:teuthology.orchestra.run.smithi089.stdout:mds.foofs.smithi195.hijxre smithi195 running (7m) 33s ago 7m 16.7M - 17.2.0 e1d6a67b021e 7bc0c6542d12 2024-09-21T04:15:55.630 INFO:teuthology.orchestra.run.smithi089.stdout:mgr.smithi089.srlyxf smithi089 *:8443,9283,8765 running (3m) 30s ago 14m 509M - 19.3.0-5103-gf36283e7 48ed0a2cf89d 3cbeaaa88241 2024-09-21T04:15:55.630 INFO:teuthology.orchestra.run.smithi089.stdout:mgr.smithi195.cgotzm smithi195 *:8443,9283,8765 running (2m) 33s ago 12m 454M - 19.3.0-5103-gf36283e7 48ed0a2cf89d d3734574cf66 2024-09-21T04:15:55.630 INFO:teuthology.orchestra.run.smithi089.stdout:mon.smithi089 smithi089 running (2m) 30s ago 14m 44.4M 2048M 19.3.0-5103-gf36283e7 48ed0a2cf89d 7dd34c8a6f21 2024-09-21T04:15:55.630 INFO:teuthology.orchestra.run.smithi089.stdout:mon.smithi195 smithi195 running (96s) 33s ago 12m 36.7M 2048M 19.3.0-5103-gf36283e7 48ed0a2cf89d 4255937444e7 2024-09-21T04:15:55.630 INFO:teuthology.orchestra.run.smithi089.stdout:nfs.foo.0.0.smithi089.dcjmqc smithi089 *:2049 running (6m) 30s ago 6m 277M - 4.0 e1d6a67b021e 98550f94e773 2024-09-21T04:15:55.631 INFO:teuthology.orchestra.run.smithi089.stdout:nfs.foo.1.0.smithi195.vhoayp smithi195 *:2049 running (6m) 33s ago 6m 88.0M - 4.0 e1d6a67b021e c60a4040130a 2024-09-21T04:15:55.631 INFO:teuthology.orchestra.run.smithi089.stdout:node-exporter.smithi089 smithi089 *:9100 running (3m) 30s ago 12m 9.92M - 1.7.0 72c9c2088986 7ef0d1588475 2024-09-21T04:15:55.631 INFO:teuthology.orchestra.run.smithi089.stdout:node-exporter.smithi195 smithi195 *:9100 running (3m) 33s ago 12m 10.7M - 1.7.0 72c9c2088986 61eafd61bc6c 2024-09-21T04:15:55.631 INFO:teuthology.orchestra.run.smithi089.stdout:osd.0 smithi089 starting - - - 1935M 2024-09-21T04:15:55.631 INFO:teuthology.orchestra.run.smithi089.stdout:osd.1 smithi089 running (11m) 30s ago 11m 480M 1935M 17.2.0 e1d6a67b021e f9ec697d08d0 2024-09-21T04:15:55.631 INFO:teuthology.orchestra.run.smithi089.stdout:osd.2 smithi089 running (10m) 30s ago 10m 393M 1935M 17.2.0 e1d6a67b021e cfa000948859 2024-09-21T04:15:55.631 INFO:teuthology.orchestra.run.smithi089.stdout:osd.3 smithi089 running (10m) 30s ago 10m 500M 1935M 17.2.0 e1d6a67b021e 31c610ed5815 2024-09-21T04:15:55.631 INFO:teuthology.orchestra.run.smithi089.stdout:osd.4 smithi195 running (9m) 33s ago 9m 524M 2704M 17.2.0 e1d6a67b021e f00e18a1f220 2024-09-21T04:15:55.631 INFO:teuthology.orchestra.run.smithi089.stdout:osd.5 smithi195 running (9m) 33s ago 9m 318M 2704M 17.2.0 e1d6a67b021e cecfd6ff1eee 2024-09-21T04:15:55.631 INFO:teuthology.orchestra.run.smithi089.stdout:osd.6 smithi195 running (9m) 33s ago 9m 510M 2704M 17.2.0 e1d6a67b021e 0d7db362cc23 2024-09-21T04:15:55.631 INFO:teuthology.orchestra.run.smithi089.stdout:osd.7 smithi195 running (8m) 33s ago 8m 387M 2704M 17.2.0 e1d6a67b021e 0109b3fea188 2024-09-21T04:15:55.631 INFO:teuthology.orchestra.run.smithi089.stdout:prometheus.smithi089 smithi089 *:9095 running (2m) 30s ago 12m 46.4M - 2.51.0 1d3b7f56885b 8c9f309a6459 2024-09-21T04:15:56.202 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:15:56 smithi089 ceph-mon[64076]: from='client.34204 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T04:15:56.202 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:15:56 smithi089 ceph-mon[64076]: from='client.34208 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T04:15:56.212 INFO:teuthology.orchestra.run.smithi089.stdout:{ 2024-09-21T04:15:56.212 INFO:teuthology.orchestra.run.smithi089.stdout: "mon": { 2024-09-21T04:15:56.212 INFO:teuthology.orchestra.run.smithi089.stdout: "ceph version 19.3.0-5103-gf36283e7 (f36283e7c91cac1c01cb9a7c4ff21bd697dbd4f2) squid (dev)": 2 2024-09-21T04:15:56.212 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2024-09-21T04:15:56.212 INFO:teuthology.orchestra.run.smithi089.stdout: "mgr": { 2024-09-21T04:15:56.212 INFO:teuthology.orchestra.run.smithi089.stdout: "ceph version 19.3.0-5103-gf36283e7 (f36283e7c91cac1c01cb9a7c4ff21bd697dbd4f2) squid (dev)": 2 2024-09-21T04:15:56.213 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2024-09-21T04:15:56.213 INFO:teuthology.orchestra.run.smithi089.stdout: "osd": { 2024-09-21T04:15:56.213 INFO:teuthology.orchestra.run.smithi089.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 7 2024-09-21T04:15:56.213 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2024-09-21T04:15:56.213 INFO:teuthology.orchestra.run.smithi089.stdout: "mds": { 2024-09-21T04:15:56.213 INFO:teuthology.orchestra.run.smithi089.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-21T04:15:56.213 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2024-09-21T04:15:56.213 INFO:teuthology.orchestra.run.smithi089.stdout: "overall": { 2024-09-21T04:15:56.213 INFO:teuthology.orchestra.run.smithi089.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 9, 2024-09-21T04:15:56.213 INFO:teuthology.orchestra.run.smithi089.stdout: "ceph version 19.3.0-5103-gf36283e7 (f36283e7c91cac1c01cb9a7c4ff21bd697dbd4f2) squid (dev)": 4 2024-09-21T04:15:56.213 INFO:teuthology.orchestra.run.smithi089.stdout: } 2024-09-21T04:15:56.213 INFO:teuthology.orchestra.run.smithi089.stdout:} 2024-09-21T04:15:56.354 INFO:teuthology.orchestra.run.smithi089.stdout: 5 170474 10.40 MB/sec execute 275 sec latency 47.025 ms 2024-09-21T04:15:56.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:15:56 smithi195 ceph-mon[49435]: from='client.34204 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T04:15:56.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:15:56 smithi195 ceph-mon[49435]: from='client.34208 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T04:15:56.672 INFO:teuthology.orchestra.run.smithi089.stdout:{ 2024-09-21T04:15:56.672 INFO:teuthology.orchestra.run.smithi089.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:2195e5f178be0a9390728e2a14088236d8fe6f82635f5c84418fe37a04457e00", 2024-09-21T04:15:56.672 INFO:teuthology.orchestra.run.smithi089.stdout: "in_progress": true, 2024-09-21T04:15:56.672 INFO:teuthology.orchestra.run.smithi089.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-21T04:15:56.672 INFO:teuthology.orchestra.run.smithi089.stdout: "services_complete": [ 2024-09-21T04:15:56.672 INFO:teuthology.orchestra.run.smithi089.stdout: "mgr", 2024-09-21T04:15:56.672 INFO:teuthology.orchestra.run.smithi089.stdout: "crash", 2024-09-21T04:15:56.672 INFO:teuthology.orchestra.run.smithi089.stdout: "mon" 2024-09-21T04:15:56.672 INFO:teuthology.orchestra.run.smithi089.stdout: ], 2024-09-21T04:15:56.672 INFO:teuthology.orchestra.run.smithi089.stdout: "progress": "6/23 daemons upgraded", 2024-09-21T04:15:56.673 INFO:teuthology.orchestra.run.smithi089.stdout: "message": "Currently upgrading osd daemons", 2024-09-21T04:15:56.673 INFO:teuthology.orchestra.run.smithi089.stdout: "is_paused": false 2024-09-21T04:15:56.673 INFO:teuthology.orchestra.run.smithi089.stdout:} 2024-09-21T04:15:57.228 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:15:57 smithi089 ceph-mon[64076]: from='client.34212 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T04:15:57.228 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:15:57 smithi089 ceph-mon[64076]: pgmap v62: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 323 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 6.4 KiB/s rd, 4.0 MiB/s wr, 411 op/s; 219/1593 objects degraded (13.748%) 2024-09-21T04:15:57.228 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:15:57 smithi089 ceph-mon[64076]: from='client.? 172.21.15.89:0/2795242131' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T04:15:57.303 INFO:teuthology.orchestra.run.smithi089.stdout:HEALTH_WARN 1 osds down; Degraded data redundancy: 219/1593 objects degraded (13.748%), 29 pgs degraded 2024-09-21T04:15:57.304 INFO:teuthology.orchestra.run.smithi089.stdout:[WRN] OSD_DOWN: 1 osds down 2024-09-21T04:15:57.304 INFO:teuthology.orchestra.run.smithi089.stdout: osd.0 (root=default,host=smithi089) is down 2024-09-21T04:15:57.304 INFO:teuthology.orchestra.run.smithi089.stdout:[WRN] PG_DEGRADED: Degraded data redundancy: 219/1593 objects degraded (13.748%), 29 pgs degraded 2024-09-21T04:15:57.304 INFO:teuthology.orchestra.run.smithi089.stdout: pg 1.0 is active+undersized+degraded, acting [7,6] 2024-09-21T04:15:57.304 INFO:teuthology.orchestra.run.smithi089.stdout: pg 2.0 is active+undersized+degraded, acting [7,1] 2024-09-21T04:15:57.304 INFO:teuthology.orchestra.run.smithi089.stdout: pg 2.1 is active+undersized+degraded, acting [2,3] 2024-09-21T04:15:57.304 INFO:teuthology.orchestra.run.smithi089.stdout: pg 2.4 is active+undersized+degraded, acting [1,7] 2024-09-21T04:15:57.304 INFO:teuthology.orchestra.run.smithi089.stdout: pg 2.5 is active+undersized+degraded, acting [7,4] 2024-09-21T04:15:57.304 INFO:teuthology.orchestra.run.smithi089.stdout: pg 2.8 is active+undersized+degraded, acting [7,1] 2024-09-21T04:15:57.304 INFO:teuthology.orchestra.run.smithi089.stdout: pg 2.c is active+undersized+degraded, acting [2,5] 2024-09-21T04:15:57.304 INFO:teuthology.orchestra.run.smithi089.stdout: pg 2.f is active+undersized+degraded, acting [4,7] 2024-09-21T04:15:57.304 INFO:teuthology.orchestra.run.smithi089.stdout: pg 2.10 is active+undersized+degraded, acting [2,1] 2024-09-21T04:15:57.304 INFO:teuthology.orchestra.run.smithi089.stdout: pg 2.13 is active+undersized+degraded, acting [4,3] 2024-09-21T04:15:57.304 INFO:teuthology.orchestra.run.smithi089.stdout: pg 2.15 is active+undersized+degraded, acting [1,5] 2024-09-21T04:15:57.305 INFO:teuthology.orchestra.run.smithi089.stdout: pg 2.19 is active+undersized+degraded, acting [3,6] 2024-09-21T04:15:57.305 INFO:teuthology.orchestra.run.smithi089.stdout: pg 2.1d is active+undersized+degraded, acting [7,6] 2024-09-21T04:15:57.305 INFO:teuthology.orchestra.run.smithi089.stdout: pg 2.1e is active+undersized+degraded, acting [3,5] 2024-09-21T04:15:57.305 INFO:teuthology.orchestra.run.smithi089.stdout: pg 2.1f is active+undersized+degraded, acting [7,4] 2024-09-21T04:15:57.305 INFO:teuthology.orchestra.run.smithi089.stdout: pg 3.1 is active+undersized+degraded, acting [4,3] 2024-09-21T04:15:57.305 INFO:teuthology.orchestra.run.smithi089.stdout: pg 3.3 is active+undersized+degraded, acting [4,6] 2024-09-21T04:15:57.305 INFO:teuthology.orchestra.run.smithi089.stdout: pg 3.6 is active+undersized+degraded, acting [1,4] 2024-09-21T04:15:57.305 INFO:teuthology.orchestra.run.smithi089.stdout: pg 3.7 is active+undersized+degraded, acting [3,7] 2024-09-21T04:15:57.305 INFO:teuthology.orchestra.run.smithi089.stdout: pg 3.b is active+undersized+degraded, acting [3,4] 2024-09-21T04:15:57.305 INFO:teuthology.orchestra.run.smithi089.stdout: pg 3.f is active+undersized+degraded, acting [7,4] 2024-09-21T04:15:57.305 INFO:teuthology.orchestra.run.smithi089.stdout: pg 3.10 is active+undersized+degraded, acting [6,5] 2024-09-21T04:15:57.305 INFO:teuthology.orchestra.run.smithi089.stdout: pg 3.12 is active+undersized+degraded, acting [7,3] 2024-09-21T04:15:57.305 INFO:teuthology.orchestra.run.smithi089.stdout: pg 3.17 is active+undersized+degraded, acting [5,3] 2024-09-21T04:15:57.305 INFO:teuthology.orchestra.run.smithi089.stdout: pg 3.18 is active+undersized+degraded, acting [3,1] 2024-09-21T04:15:57.305 INFO:teuthology.orchestra.run.smithi089.stdout: pg 3.1b is active+undersized+degraded, acting [4,7] 2024-09-21T04:15:57.305 INFO:teuthology.orchestra.run.smithi089.stdout: pg 3.1f is active+undersized+degraded, acting [5,2] 2024-09-21T04:15:57.306 INFO:teuthology.orchestra.run.smithi089.stdout: pg 4.0 is active+undersized+degraded, acting [3,7] 2024-09-21T04:15:57.306 INFO:teuthology.orchestra.run.smithi089.stdout: pg 4.19 is active+undersized+degraded, acting [3,2] 2024-09-21T04:15:57.354 INFO:teuthology.orchestra.run.smithi089.stdout: 5 170691 10.37 MB/sec execute 276 sec latency 53.578 ms 2024-09-21T04:15:57.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:15:57 smithi195 ceph-mon[49435]: from='client.34212 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T04:15:57.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:15:57 smithi195 ceph-mon[49435]: pgmap v62: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 323 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 6.4 KiB/s rd, 4.0 MiB/s wr, 411 op/s; 219/1593 objects degraded (13.748%) 2024-09-21T04:15:57.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:15:57 smithi195 ceph-mon[49435]: from='client.? 172.21.15.89:0/2795242131' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T04:15:58.245 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:15:58 smithi089 ceph-mon[64076]: from='client.34220 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T04:15:58.245 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:15:58 smithi089 ceph-mon[64076]: from='client.? 172.21.15.89:0/1026544646' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-21T04:15:58.355 INFO:teuthology.orchestra.run.smithi089.stdout: 5 171027 10.34 MB/sec execute 277 sec latency 31.962 ms 2024-09-21T04:15:58.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:15:58 smithi195 ceph-mon[49435]: from='client.34220 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T04:15:58.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:15:58 smithi195 ceph-mon[49435]: from='client.? 172.21.15.89:0/1026544646' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-21T04:15:59.264 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:15:59 smithi089 ceph-mon[64076]: pgmap v63: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 320 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 6.4 KiB/s rd, 4.4 MiB/s wr, 426 op/s; 219/1617 objects degraded (13.544%) 2024-09-21T04:15:59.355 INFO:teuthology.orchestra.run.smithi089.stdout: 5 171312 10.32 MB/sec execute 278 sec latency 58.377 ms 2024-09-21T04:15:59.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:15:59 smithi195 ceph-mon[49435]: pgmap v63: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 320 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 6.4 KiB/s rd, 4.4 MiB/s wr, 426 op/s; 219/1617 objects degraded (13.544%) 2024-09-21T04:16:00.284 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:16:00 smithi089 ceph-mon[64076]: Health check update: Degraded data redundancy: 219/1617 objects degraded (13.544%), 29 pgs degraded (PG_DEGRADED) 2024-09-21T04:16:00.355 INFO:teuthology.orchestra.run.smithi089.stdout: 5 171610 10.32 MB/sec execute 279 sec latency 76.984 ms 2024-09-21T04:16:00.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:16:00 smithi195 ceph-mon[49435]: Health check update: Degraded data redundancy: 219/1617 objects degraded (13.544%), 29 pgs degraded (PG_DEGRADED) 2024-09-21T04:16:01.050 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:16:01 smithi089 ceph-mon[64076]: pgmap v64: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 332 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 8.6 KiB/s rd, 5.9 MiB/s wr, 572 op/s; 227/1701 objects degraded (13.345%) 2024-09-21T04:16:01.050 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:16:01 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:16:01.050 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:16:01 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:16:01.355 INFO:teuthology.orchestra.run.smithi089.stdout: 5 171949 10.31 MB/sec execute 280 sec latency 46.296 ms 2024-09-21T04:16:01.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:16:01 smithi195 ceph-mon[49435]: pgmap v64: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 332 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 8.6 KiB/s rd, 5.9 MiB/s wr, 572 op/s; 227/1701 objects degraded (13.345%) 2024-09-21T04:16:01.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:16:01 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:16:01.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:16:01 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:16:02.356 INFO:teuthology.orchestra.run.smithi089.stdout: 5 172275 10.30 MB/sec execute 281 sec latency 74.654 ms 2024-09-21T04:16:02.601 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:16:02 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:16:02.601 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:16:02 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:16:02.601 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:16:02 smithi089 ceph-mon[64076]: pgmap v65: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 332 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 5.5 KiB/s rd, 3.9 MiB/s wr, 380 op/s; 227/1701 objects degraded (13.345%) 2024-09-21T04:16:02.601 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:16:02 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T04:16:02.859 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:16:02 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:16:02.859 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:16:02 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:16:02.859 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:16:02 smithi195 ceph-mon[49435]: pgmap v65: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 332 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 5.5 KiB/s rd, 3.9 MiB/s wr, 380 op/s; 227/1701 objects degraded (13.345%) 2024-09-21T04:16:02.859 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:16:02 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T04:16:02.861 INFO:journalctl@ceph.osd.0.smithi089.stdout:Sep 21 04:16:02 smithi089 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-0[68909]: 2024-09-21T04:16:02.664+0000 7f4f122fe640 -1 osd.0 0 read_superblock omap replica is missing. 2024-09-21T04:16:03.355 INFO:teuthology.orchestra.run.smithi089.stdout: 5 172559 10.27 MB/sec execute 282 sec latency 135.358 ms 2024-09-21T04:16:03.900 INFO:journalctl@ceph.osd.0.smithi089.stdout:Sep 21 04:16:03 smithi089 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-0[68909]: 2024-09-21T04:16:03.828+0000 7f4f122fe640 -1 osd.0 74 log_to_monitors true 2024-09-21T04:16:03.901 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:16:03 smithi089 ceph-mon[64076]: from='osd.0 [v2:172.21.15.89:6802/757070544,v1:172.21.15.89:6803/757070544]' entity='osd.0' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]} : dispatch 2024-09-21T04:16:04.150 INFO:journalctl@ceph.osd.0.smithi089.stdout:Sep 21 04:16:03 smithi089 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-0[68909]: 2024-09-21T04:16:03.969+0000 7f4f0a0a8640 -1 osd.0 74 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-21T04:16:04.355 INFO:teuthology.orchestra.run.smithi089.stdout: 5 172840 10.27 MB/sec execute 283 sec latency 71.005 ms 2024-09-21T04:16:04.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:16:03 smithi195 ceph-mon[49435]: from='osd.0 [v2:172.21.15.89:6802/757070544,v1:172.21.15.89:6803/757070544]' entity='osd.0' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]} : dispatch 2024-09-21T04:16:04.924 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:16:04 smithi089 ceph-mon[64076]: from='osd.0 [v2:172.21.15.89:6802/757070544,v1:172.21.15.89:6803/757070544]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-09-21T04:16:04.924 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:16:04 smithi089 ceph-mon[64076]: osdmap e77: 8 total, 7 up, 8 in 2024-09-21T04:16:04.924 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:16:04 smithi089 ceph-mon[64076]: from='osd.0 [v2:172.21.15.89:6802/757070544,v1:172.21.15.89:6803/757070544]' entity='osd.0' cmd={"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi089", "root=default"]} : dispatch 2024-09-21T04:16:04.924 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:16:04 smithi089 ceph-mon[64076]: pgmap v67: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 320 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.3 KiB/s rd, 4.1 MiB/s wr, 433 op/s; 200/1539 objects degraded (12.995%) 2024-09-21T04:16:04.924 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:16:04 smithi089 ceph-mon[64076]: Health check update: Degraded data redundancy: 227/1701 objects degraded (13.345%), 29 pgs degraded (PG_DEGRADED) 2024-09-21T04:16:05.355 INFO:teuthology.orchestra.run.smithi089.stdout: 5 172965 10.25 MB/sec execute 284 sec latency 455.343 ms 2024-09-21T04:16:05.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:16:04 smithi195 ceph-mon[49435]: from='osd.0 [v2:172.21.15.89:6802/757070544,v1:172.21.15.89:6803/757070544]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-09-21T04:16:05.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:16:04 smithi195 ceph-mon[49435]: osdmap e77: 8 total, 7 up, 8 in 2024-09-21T04:16:05.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:16:04 smithi195 ceph-mon[49435]: from='osd.0 [v2:172.21.15.89:6802/757070544,v1:172.21.15.89:6803/757070544]' entity='osd.0' cmd={"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi089", "root=default"]} : dispatch 2024-09-21T04:16:05.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:16:04 smithi195 ceph-mon[49435]: pgmap v67: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 320 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.3 KiB/s rd, 4.1 MiB/s wr, 433 op/s; 200/1539 objects degraded (12.995%) 2024-09-21T04:16:05.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:16:04 smithi195 ceph-mon[49435]: Health check update: Degraded data redundancy: 227/1701 objects degraded (13.345%), 29 pgs degraded (PG_DEGRADED) 2024-09-21T04:16:06.015 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:16:05 smithi089 ceph-mon[64076]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-21T04:16:06.015 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:16:05 smithi089 ceph-mon[64076]: osd.0 [v2:172.21.15.89:6802/757070544,v1:172.21.15.89:6803/757070544] boot 2024-09-21T04:16:06.015 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:16:05 smithi089 ceph-mon[64076]: osdmap e78: 8 total, 8 up, 8 in 2024-09-21T04:16:06.015 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:16:05 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "osd metadata", "id": 0} : dispatch 2024-09-21T04:16:06.015 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:16:05 smithi089 ceph-mon[64076]: osdmap e79: 8 total, 8 up, 8 in 2024-09-21T04:16:06.355 INFO:teuthology.orchestra.run.smithi089.stdout: 5 172965 10.21 MB/sec execute 285 sec latency 1455.393 ms 2024-09-21T04:16:06.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:16:05 smithi195 ceph-mon[49435]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-21T04:16:06.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:16:05 smithi195 ceph-mon[49435]: osd.0 [v2:172.21.15.89:6802/757070544,v1:172.21.15.89:6803/757070544] boot 2024-09-21T04:16:06.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:16:05 smithi195 ceph-mon[49435]: osdmap e78: 8 total, 8 up, 8 in 2024-09-21T04:16:06.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:16:05 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "osd metadata", "id": 0} : dispatch 2024-09-21T04:16:06.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:16:05 smithi195 ceph-mon[49435]: osdmap e79: 8 total, 8 up, 8 in 2024-09-21T04:16:06.982 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:16:06 smithi089 ceph-mon[64076]: pgmap v70: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 320 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.8 KiB/s rd, 2.8 MiB/s wr, 393 op/s; 200/1539 objects degraded (12.995%) 2024-09-21T04:16:06.982 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:16:06 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:16:06.982 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:16:06 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:16:06.982 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:16:06 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T04:16:06.982 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:16:06 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T04:16:06.982 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:16:06 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:16:06.982 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:16:06 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:16:06.982 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:16:06 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T04:16:06.982 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:16:06 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "versions"} : dispatch 2024-09-21T04:16:06.983 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:16:06 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "versions"} : dispatch 2024-09-21T04:16:06.983 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:16:06 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "versions"} : dispatch 2024-09-21T04:16:06.983 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:16:06 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16} : dispatch 2024-09-21T04:16:06.983 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:16:06 smithi089 ceph-mon[64076]: osdmap e80: 8 total, 8 up, 8 in 2024-09-21T04:16:07.260 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:16:06 smithi195 ceph-mon[49435]: pgmap v70: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 320 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.8 KiB/s rd, 2.8 MiB/s wr, 393 op/s; 200/1539 objects degraded (12.995%) 2024-09-21T04:16:07.260 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:16:06 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:16:07.260 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:16:06 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:16:07.260 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:16:06 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T04:16:07.260 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:16:06 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T04:16:07.260 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:16:06 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:16:07.260 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:16:06 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:16:07.260 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:16:06 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T04:16:07.260 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:16:06 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "versions"} : dispatch 2024-09-21T04:16:07.261 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:16:06 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "versions"} : dispatch 2024-09-21T04:16:07.261 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:16:06 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "versions"} : dispatch 2024-09-21T04:16:07.261 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:16:06 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16} : dispatch 2024-09-21T04:16:07.261 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:16:06 smithi195 ceph-mon[49435]: osdmap e80: 8 total, 8 up, 8 in 2024-09-21T04:16:07.356 INFO:teuthology.orchestra.run.smithi089.stdout: 5 173117 10.20 MB/sec execute 286 sec latency 2070.417 ms 2024-09-21T04:16:08.264 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:16:07 smithi089 ceph-mon[64076]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-09-21T04:16:08.265 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:16:07 smithi089 ceph-mon[64076]: Upgrade: unsafe to stop osd(s) at this time (10 PGs are or would become offline) 2024-09-21T04:16:08.275 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:16:07 smithi195 ceph-mon[49435]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-09-21T04:16:08.275 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:16:07 smithi195 ceph-mon[49435]: Upgrade: unsafe to stop osd(s) at this time (10 PGs are or would become offline) 2024-09-21T04:16:08.356 INFO:teuthology.orchestra.run.smithi089.stdout: 5 173342 10.20 MB/sec execute 287 sec latency 431.273 ms 2024-09-21T04:16:09.109 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:16:08 smithi195 ceph-mon[49435]: osdmap e81: 8 total, 8 up, 8 in 2024-09-21T04:16:09.109 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:16:08 smithi195 ceph-mon[49435]: pgmap v73: 97 pgs: 3 active+undersized+remapped, 7 active+undersized, 22 active+undersized+degraded, 65 active+clean; 323 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 1.4 MiB/s wr, 43 op/s; 156/1542 objects degraded (10.117%); 1024 KiB/s, 11 objects/s recovering 2024-09-21T04:16:09.192 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:16:08 smithi089 ceph-mon[64076]: osdmap e81: 8 total, 8 up, 8 in 2024-09-21T04:16:09.192 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:16:08 smithi089 ceph-mon[64076]: pgmap v73: 97 pgs: 3 active+undersized+remapped, 7 active+undersized, 22 active+undersized+degraded, 65 active+clean; 323 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 1.4 MiB/s wr, 43 op/s; 156/1542 objects degraded (10.117%); 1024 KiB/s, 11 objects/s recovering 2024-09-21T04:16:09.356 INFO:teuthology.orchestra.run.smithi089.stdout: 5 173489 10.19 MB/sec execute 288 sec latency 1037.483 ms 2024-09-21T04:16:09.979 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:16:09 smithi089 ceph-mon[64076]: osdmap e82: 8 total, 8 up, 8 in 2024-09-21T04:16:10.043 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:16:09 smithi089 ceph-mon[64076]: Health check update: Degraded data redundancy: 156/1542 objects degraded (10.117%), 22 pgs degraded (PG_DEGRADED) 2024-09-21T04:16:10.356 INFO:teuthology.orchestra.run.smithi089.stdout: 5 173746 10.19 MB/sec execute 289 sec latency 971.418 ms 2024-09-21T04:16:10.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:16:09 smithi195 ceph-mon[49435]: osdmap e82: 8 total, 8 up, 8 in 2024-09-21T04:16:10.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:16:09 smithi195 ceph-mon[49435]: Health check update: Degraded data redundancy: 156/1542 objects degraded (10.117%), 22 pgs degraded (PG_DEGRADED) 2024-09-21T04:16:10.992 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:16:10 smithi089 ceph-mon[64076]: pgmap v75: 97 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 7 peering, 4 active+undersized+remapped, 82 active+clean; 327 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 3.0 KiB/s rd, 4.4 MiB/s wr, 322 op/s; 93/1566 objects degraded (5.939%); 2.2 MiB/s, 38 objects/s recovering 2024-09-21T04:16:10.992 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:16:10 smithi089 ceph-mon[64076]: osdmap e83: 8 total, 8 up, 8 in 2024-09-21T04:16:11.356 INFO:teuthology.orchestra.run.smithi089.stdout: 5 174027 10.19 MB/sec execute 290 sec latency 44.442 ms 2024-09-21T04:16:11.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:16:10 smithi195 ceph-mon[49435]: pgmap v75: 97 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 7 peering, 4 active+undersized+remapped, 82 active+clean; 327 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 3.0 KiB/s rd, 4.4 MiB/s wr, 322 op/s; 93/1566 objects degraded (5.939%); 2.2 MiB/s, 38 objects/s recovering 2024-09-21T04:16:11.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:16:10 smithi195 ceph-mon[49435]: osdmap e83: 8 total, 8 up, 8 in 2024-09-21T04:16:12.356 INFO:teuthology.orchestra.run.smithi089.stdout: 5 174336 10.17 MB/sec execute 291 sec latency 61.960 ms 2024-09-21T04:16:13.063 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:16:12 smithi089 ceph-mon[64076]: pgmap v77: 97 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 7 peering, 4 active+undersized+remapped, 82 active+clean; 327 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 2.4 KiB/s rd, 3.5 MiB/s wr, 258 op/s; 93/1566 objects degraded (5.939%); 1.8 MiB/s, 30 objects/s recovering 2024-09-21T04:16:13.356 INFO:teuthology.orchestra.run.smithi089.stdout: 5 174653 10.14 MB/sec execute 292 sec latency 67.784 ms 2024-09-21T04:16:13.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:16:12 smithi195 ceph-mon[49435]: pgmap v77: 97 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 7 peering, 4 active+undersized+remapped, 82 active+clean; 327 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 2.4 KiB/s rd, 3.5 MiB/s wr, 258 op/s; 93/1566 objects degraded (5.939%); 1.8 MiB/s, 30 objects/s recovering 2024-09-21T04:16:14.100 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:16:13 smithi089 ceph-mon[64076]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 93/1566 objects degraded (5.939%), 4 pgs degraded) 2024-09-21T04:16:14.101 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:16:13 smithi089 ceph-mon[64076]: Cluster is now healthy 2024-09-21T04:16:14.356 INFO:teuthology.orchestra.run.smithi089.stdout: 5 174909 10.12 MB/sec execute 293 sec latency 67.201 ms 2024-09-21T04:16:14.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:16:13 smithi195 ceph-mon[49435]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 93/1566 objects degraded (5.939%), 4 pgs degraded) 2024-09-21T04:16:14.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:16:13 smithi195 ceph-mon[49435]: Cluster is now healthy 2024-09-21T04:16:15.141 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:16:14 smithi089 ceph-mon[64076]: pgmap v78: 97 pgs: 97 active+clean; 334 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 4.5 KiB/s rd, 6.2 MiB/s wr, 508 op/s; 1.5 MiB/s, 48 objects/s recovering 2024-09-21T04:16:15.357 INFO:teuthology.orchestra.run.smithi089.stdout: 5 175259 10.13 MB/sec execute 294 sec latency 62.278 ms 2024-09-21T04:16:15.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:16:14 smithi195 ceph-mon[49435]: pgmap v78: 97 pgs: 97 active+clean; 334 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 4.5 KiB/s rd, 6.2 MiB/s wr, 508 op/s; 1.5 MiB/s, 48 objects/s recovering 2024-09-21T04:16:16.365 INFO:teuthology.orchestra.run.smithi089.stdout: 5 175853 10.13 MB/sec execute 295 sec latency 47.235 ms 2024-09-21T04:16:17.179 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:16:17 smithi089 ceph-mon[64076]: pgmap v79: 97 pgs: 97 active+clean; 334 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 3.4 KiB/s rd, 4.0 MiB/s wr, 360 op/s; 643 KiB/s, 30 objects/s recovering 2024-09-21T04:16:17.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:16:17 smithi195 ceph-mon[49435]: pgmap v79: 97 pgs: 97 active+clean; 334 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 3.4 KiB/s rd, 4.0 MiB/s wr, 360 op/s; 643 KiB/s, 30 objects/s recovering 2024-09-21T04:16:17.365 INFO:teuthology.orchestra.run.smithi089.stdout: 5 176181 10.11 MB/sec execute 296 sec latency 54.174 ms 2024-09-21T04:16:18.365 INFO:teuthology.orchestra.run.smithi089.stdout: 5 176573 10.14 MB/sec execute 297 sec latency 53.469 ms 2024-09-21T04:16:18.451 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:16:18 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:16:18.451 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:16:18 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T04:16:18.451 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:16:18 smithi089 ceph-mon[64076]: pgmap v80: 97 pgs: 97 active+clean; 334 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 1.7 KiB/s rd, 3.3 MiB/s wr, 361 op/s; 0 B/s, 15 objects/s recovering 2024-09-21T04:16:18.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:16:18 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:16:18.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:16:18 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T04:16:18.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:16:18 smithi195 ceph-mon[49435]: pgmap v80: 97 pgs: 97 active+clean; 334 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 1.7 KiB/s rd, 3.3 MiB/s wr, 361 op/s; 0 B/s, 15 objects/s recovering 2024-09-21T04:16:19.365 INFO:teuthology.orchestra.run.smithi089.stdout: 5 177166 10.18 MB/sec execute 298 sec latency 36.519 ms 2024-09-21T04:16:20.365 INFO:teuthology.orchestra.run.smithi089.stdout: 5 177584 10.20 MB/sec execute 299 sec latency 26.581 ms 2024-09-21T04:16:21.223 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:16:20 smithi089 ceph-mon[64076]: pgmap v81: 97 pgs: 97 active+clean; 322 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.4 MiB/s wr, 457 op/s; 0 B/s, 13 objects/s recovering 2024-09-21T04:16:21.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:16:20 smithi195 ceph-mon[49435]: pgmap v81: 97 pgs: 97 active+clean; 322 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.4 MiB/s wr, 457 op/s; 0 B/s, 13 objects/s recovering 2024-09-21T04:16:21.365 INFO:teuthology.orchestra.run.smithi089.stdout: 5 177863 10.18 MB/sec execute 300 sec latency 44.059 ms 2024-09-21T04:16:22.235 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:16:22 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16} : dispatch 2024-09-21T04:16:22.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:16:22 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16} : dispatch 2024-09-21T04:16:22.365 INFO:teuthology.orchestra.run.smithi089.stdout: 5 178154 10.15 MB/sec execute 301 sec latency 45.642 ms 2024-09-21T04:16:23.265 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:16:23 smithi089 ceph-mon[64076]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-09-21T04:16:23.265 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:16:23 smithi089 ceph-mon[64076]: Upgrade: osd.1 is safe to restart 2024-09-21T04:16:23.265 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:16:23 smithi089 ceph-mon[64076]: pgmap v82: 97 pgs: 97 active+clean; 322 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 5.6 KiB/s rd, 3.6 MiB/s wr, 381 op/s; 0 B/s, 11 objects/s recovering 2024-09-21T04:16:23.265 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:16:23 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:16:23.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:16:23 smithi195 ceph-mon[49435]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-09-21T04:16:23.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:16:23 smithi195 ceph-mon[49435]: Upgrade: osd.1 is safe to restart 2024-09-21T04:16:23.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:16:23 smithi195 ceph-mon[49435]: pgmap v82: 97 pgs: 97 active+clean; 322 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 5.6 KiB/s rd, 3.6 MiB/s wr, 381 op/s; 0 B/s, 11 objects/s recovering 2024-09-21T04:16:23.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:16:23 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:16:23.365 INFO:teuthology.orchestra.run.smithi089.stdout: 5 178469 10.15 MB/sec execute 302 sec latency 51.981 ms 2024-09-21T04:16:24.365 INFO:teuthology.orchestra.run.smithi089.stdout: 5 178819 10.15 MB/sec execute 303 sec latency 53.728 ms 2024-09-21T04:16:25.223 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:16:25 smithi089 ceph-mon[64076]: pgmap v83: 97 pgs: 97 active+clean; 332 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 8.5 KiB/s rd, 6.4 MiB/s wr, 619 op/s; 0 B/s, 11 objects/s recovering 2024-09-21T04:16:25.223 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:16:25 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:16:25.223 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:16:25 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "auth get", "entity": "osd.1"} : dispatch 2024-09-21T04:16:25.223 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:16:25 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T04:16:25.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:16:25 smithi195 ceph-mon[49435]: pgmap v83: 97 pgs: 97 active+clean; 332 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 8.5 KiB/s rd, 6.4 MiB/s wr, 619 op/s; 0 B/s, 11 objects/s recovering 2024-09-21T04:16:25.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:16:25 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:16:25.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:16:25 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "auth get", "entity": "osd.1"} : dispatch 2024-09-21T04:16:25.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:16:25 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T04:16:25.366 INFO:teuthology.orchestra.run.smithi089.stdout: 5 179283 10.14 MB/sec execute 304 sec latency 67.313 ms 2024-09-21T04:16:26.232 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:16:26 smithi089 ceph-mon[64076]: Upgrade: Updating osd.1 2024-09-21T04:16:26.232 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:16:26 smithi089 ceph-mon[64076]: Deploying daemon osd.1 on smithi089 2024-09-21T04:16:26.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:16:26 smithi195 ceph-mon[49435]: Upgrade: Updating osd.1 2024-09-21T04:16:26.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:16:26 smithi195 ceph-mon[49435]: Deploying daemon osd.1 on smithi089 2024-09-21T04:16:26.365 INFO:teuthology.orchestra.run.smithi089.stdout: 5 179624 10.12 MB/sec execute 305 sec latency 64.305 ms 2024-09-21T04:16:27.267 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:16:27 smithi089 ceph-mon[64076]: pgmap v84: 97 pgs: 97 active+clean; 332 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 7.2 KiB/s rd, 4.8 MiB/s wr, 473 op/s 2024-09-21T04:16:27.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:16:27 smithi195 ceph-mon[49435]: pgmap v84: 97 pgs: 97 active+clean; 332 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 7.2 KiB/s rd, 4.8 MiB/s wr, 473 op/s 2024-09-21T04:16:27.365 INFO:teuthology.orchestra.run.smithi089.stdout: 5 179895 10.12 MB/sec execute 306 sec latency 53.954 ms 2024-09-21T04:16:27.783 INFO:teuthology.orchestra.run.smithi089.stdout:true 2024-09-21T04:16:28.366 INFO:teuthology.orchestra.run.smithi089.stdout: 5 180161 10.12 MB/sec execute 307 sec latency 58.175 ms 2024-09-21T04:16:28.873 INFO:teuthology.orchestra.run.smithi089.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-21T04:16:28.873 INFO:teuthology.orchestra.run.smithi089.stdout:alertmanager.smithi089 smithi089 *:9093,9094 running (4m) 28s ago 13m 17.2M - 0.27.0 11f11916f8cd 65dfcbcafee1 2024-09-21T04:16:28.873 INFO:teuthology.orchestra.run.smithi089.stdout:crash.smithi089 smithi089 running (79s) 28s ago 13m 6891k - 19.3.0-5103-gf36283e7 48ed0a2cf89d 6c415a5e173e 2024-09-21T04:16:28.873 INFO:teuthology.orchestra.run.smithi089.stdout:crash.smithi195 smithi195 running (72s) 67s ago 13m 6891k - 19.3.0-5103-gf36283e7 48ed0a2cf89d 78ce05a4e9ab 2024-09-21T04:16:28.874 INFO:teuthology.orchestra.run.smithi089.stdout:grafana.smithi089 smithi089 *:3000 running (4m) 28s ago 13m 47.8M - dad864ee21e9 ec4d6738bce7 2024-09-21T04:16:28.874 INFO:teuthology.orchestra.run.smithi089.stdout:mds.foofs.smithi089.osufws smithi089 running (7m) 28s ago 7m 38.2M - 17.2.0 e1d6a67b021e 664d41859bea 2024-09-21T04:16:28.874 INFO:teuthology.orchestra.run.smithi089.stdout:mds.foofs.smithi195.hijxre smithi195 running (7m) 67s ago 7m 16.7M - 17.2.0 e1d6a67b021e 7bc0c6542d12 2024-09-21T04:16:28.874 INFO:teuthology.orchestra.run.smithi089.stdout:mgr.smithi089.srlyxf smithi089 *:8443,9283,8765 running (3m) 28s ago 14m 511M - 19.3.0-5103-gf36283e7 48ed0a2cf89d 3cbeaaa88241 2024-09-21T04:16:28.874 INFO:teuthology.orchestra.run.smithi089.stdout:mgr.smithi195.cgotzm smithi195 *:8443,9283,8765 running (2m) 67s ago 13m 454M - 19.3.0-5103-gf36283e7 48ed0a2cf89d d3734574cf66 2024-09-21T04:16:28.874 INFO:teuthology.orchestra.run.smithi089.stdout:mon.smithi089 smithi089 running (2m) 28s ago 14m 48.0M 2048M 19.3.0-5103-gf36283e7 48ed0a2cf89d 7dd34c8a6f21 2024-09-21T04:16:28.874 INFO:teuthology.orchestra.run.smithi089.stdout:mon.smithi195 smithi195 running (2m) 67s ago 12m 36.7M 2048M 19.3.0-5103-gf36283e7 48ed0a2cf89d 4255937444e7 2024-09-21T04:16:28.874 INFO:teuthology.orchestra.run.smithi089.stdout:nfs.foo.0.0.smithi089.dcjmqc smithi089 *:2049 running (7m) 28s ago 7m 283M - 4.0 e1d6a67b021e 98550f94e773 2024-09-21T04:16:28.874 INFO:teuthology.orchestra.run.smithi089.stdout:nfs.foo.1.0.smithi195.vhoayp smithi195 *:2049 running (7m) 67s ago 7m 88.0M - 4.0 e1d6a67b021e c60a4040130a 2024-09-21T04:16:28.874 INFO:teuthology.orchestra.run.smithi089.stdout:node-exporter.smithi089 smithi089 *:9100 running (4m) 28s ago 13m 9919k - 1.7.0 72c9c2088986 7ef0d1588475 2024-09-21T04:16:28.874 INFO:teuthology.orchestra.run.smithi089.stdout:node-exporter.smithi195 smithi195 *:9100 running (3m) 67s ago 12m 10.7M - 1.7.0 72c9c2088986 61eafd61bc6c 2024-09-21T04:16:28.874 INFO:teuthology.orchestra.run.smithi089.stdout:osd.0 smithi089 running (37s) 28s ago 12m 11.5M 1935M 19.3.0-5103-gf36283e7 48ed0a2cf89d 3df0adea59fb 2024-09-21T04:16:28.874 INFO:teuthology.orchestra.run.smithi089.stdout:osd.1 smithi089 running (11m) 28s ago 11m 517M 1935M 17.2.0 e1d6a67b021e f9ec697d08d0 2024-09-21T04:16:28.874 INFO:teuthology.orchestra.run.smithi089.stdout:osd.2 smithi089 running (11m) 28s ago 11m 422M 1935M 17.2.0 e1d6a67b021e cfa000948859 2024-09-21T04:16:28.874 INFO:teuthology.orchestra.run.smithi089.stdout:osd.3 smithi089 running (10m) 28s ago 10m 518M 1935M 17.2.0 e1d6a67b021e 31c610ed5815 2024-09-21T04:16:28.875 INFO:teuthology.orchestra.run.smithi089.stdout:osd.4 smithi195 running (10m) 67s ago 10m 524M 2704M 17.2.0 e1d6a67b021e f00e18a1f220 2024-09-21T04:16:28.875 INFO:teuthology.orchestra.run.smithi089.stdout:osd.5 smithi195 running (10m) 67s ago 10m 318M 2704M 17.2.0 e1d6a67b021e cecfd6ff1eee 2024-09-21T04:16:28.875 INFO:teuthology.orchestra.run.smithi089.stdout:osd.6 smithi195 running (9m) 67s ago 9m 510M 2704M 17.2.0 e1d6a67b021e 0d7db362cc23 2024-09-21T04:16:28.875 INFO:teuthology.orchestra.run.smithi089.stdout:osd.7 smithi195 running (9m) 67s ago 9m 387M 2704M 17.2.0 e1d6a67b021e 0109b3fea188 2024-09-21T04:16:28.875 INFO:teuthology.orchestra.run.smithi089.stdout:prometheus.smithi089 smithi089 *:9095 running (2m) 28s ago 12m 48.7M - 2.51.0 1d3b7f56885b 8c9f309a6459 2024-09-21T04:16:29.060 INFO:journalctl@ceph.osd.1.smithi089.stdout:Sep 21 04:16:28 smithi089 systemd[1]: Stopping Ceph osd.1 for 1b9b79fc-77ce-11ef-baf5-efdc52797490... 2024-09-21T04:16:29.060 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:16:29 smithi089 ceph-mon[64076]: from='client.34230 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T04:16:29.060 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:16:29 smithi089 ceph-mon[64076]: pgmap v85: 97 pgs: 97 active+clean; 326 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 7.2 KiB/s rd, 5.5 MiB/s wr, 558 op/s 2024-09-21T04:16:29.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:16:29 smithi195 ceph-mon[49435]: from='client.34230 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T04:16:29.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:16:29 smithi195 ceph-mon[49435]: pgmap v85: 97 pgs: 97 active+clean; 326 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 7.2 KiB/s rd, 5.5 MiB/s wr, 558 op/s 2024-09-21T04:16:29.365 INFO:teuthology.orchestra.run.smithi089.stdout: 5 180591 10.15 MB/sec execute 308 sec latency 53.545 ms 2024-09-21T04:16:29.548 INFO:teuthology.orchestra.run.smithi089.stdout:{ 2024-09-21T04:16:29.548 INFO:teuthology.orchestra.run.smithi089.stdout: "mon": { 2024-09-21T04:16:29.548 INFO:teuthology.orchestra.run.smithi089.stdout: "ceph version 19.3.0-5103-gf36283e7 (f36283e7c91cac1c01cb9a7c4ff21bd697dbd4f2) squid (dev)": 2 2024-09-21T04:16:29.548 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2024-09-21T04:16:29.548 INFO:teuthology.orchestra.run.smithi089.stdout: "mgr": { 2024-09-21T04:16:29.548 INFO:teuthology.orchestra.run.smithi089.stdout: "ceph version 19.3.0-5103-gf36283e7 (f36283e7c91cac1c01cb9a7c4ff21bd697dbd4f2) squid (dev)": 2 2024-09-21T04:16:29.548 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2024-09-21T04:16:29.548 INFO:teuthology.orchestra.run.smithi089.stdout: "osd": { 2024-09-21T04:16:29.548 INFO:teuthology.orchestra.run.smithi089.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 7, 2024-09-21T04:16:29.548 INFO:teuthology.orchestra.run.smithi089.stdout: "ceph version 19.3.0-5103-gf36283e7 (f36283e7c91cac1c01cb9a7c4ff21bd697dbd4f2) squid (dev)": 1 2024-09-21T04:16:29.548 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2024-09-21T04:16:29.548 INFO:teuthology.orchestra.run.smithi089.stdout: "mds": { 2024-09-21T04:16:29.548 INFO:teuthology.orchestra.run.smithi089.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-21T04:16:29.548 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2024-09-21T04:16:29.549 INFO:teuthology.orchestra.run.smithi089.stdout: "overall": { 2024-09-21T04:16:29.549 INFO:teuthology.orchestra.run.smithi089.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 9, 2024-09-21T04:16:29.549 INFO:teuthology.orchestra.run.smithi089.stdout: "ceph version 19.3.0-5103-gf36283e7 (f36283e7c91cac1c01cb9a7c4ff21bd697dbd4f2) squid (dev)": 5 2024-09-21T04:16:29.549 INFO:teuthology.orchestra.run.smithi089.stdout: } 2024-09-21T04:16:29.549 INFO:teuthology.orchestra.run.smithi089.stdout:} 2024-09-21T04:16:29.561 INFO:journalctl@ceph.osd.1.smithi089.stdout:Sep 21 04:16:29 smithi089 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-1[37020]: 2024-09-21T04:16:29.441+0000 7f97e7f2b700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.1 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-21T04:16:29.562 INFO:journalctl@ceph.osd.1.smithi089.stdout:Sep 21 04:16:29 smithi089 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-1[37020]: 2024-09-21T04:16:29.441+0000 7f97e7f2b700 -1 osd.1 83 *** Got signal Terminated *** 2024-09-21T04:16:29.562 INFO:journalctl@ceph.osd.1.smithi089.stdout:Sep 21 04:16:29 smithi089 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-1[37020]: 2024-09-21T04:16:29.441+0000 7f97e7f2b700 -1 osd.1 83 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-21T04:16:30.029 INFO:teuthology.orchestra.run.smithi089.stdout:{ 2024-09-21T04:16:30.029 INFO:teuthology.orchestra.run.smithi089.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:2195e5f178be0a9390728e2a14088236d8fe6f82635f5c84418fe37a04457e00", 2024-09-21T04:16:30.029 INFO:teuthology.orchestra.run.smithi089.stdout: "in_progress": true, 2024-09-21T04:16:30.029 INFO:teuthology.orchestra.run.smithi089.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-21T04:16:30.029 INFO:teuthology.orchestra.run.smithi089.stdout: "services_complete": [ 2024-09-21T04:16:30.029 INFO:teuthology.orchestra.run.smithi089.stdout: "mgr", 2024-09-21T04:16:30.029 INFO:teuthology.orchestra.run.smithi089.stdout: "crash", 2024-09-21T04:16:30.029 INFO:teuthology.orchestra.run.smithi089.stdout: "mon" 2024-09-21T04:16:30.029 INFO:teuthology.orchestra.run.smithi089.stdout: ], 2024-09-21T04:16:30.029 INFO:teuthology.orchestra.run.smithi089.stdout: "progress": "7/23 daemons upgraded", 2024-09-21T04:16:30.029 INFO:teuthology.orchestra.run.smithi089.stdout: "message": "Currently upgrading osd daemons", 2024-09-21T04:16:30.029 INFO:teuthology.orchestra.run.smithi089.stdout: "is_paused": false 2024-09-21T04:16:30.030 INFO:teuthology.orchestra.run.smithi089.stdout:} 2024-09-21T04:16:30.070 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:16:30 smithi089 ceph-mon[64076]: from='client.34234 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T04:16:30.070 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:16:30 smithi089 ceph-mon[64076]: from='client.34238 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T04:16:30.070 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:16:30 smithi089 ceph-mon[64076]: osd.1 marked itself down and dead 2024-09-21T04:16:30.070 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:16:30 smithi089 ceph-mon[64076]: from='client.? 172.21.15.89:0/1809051875' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T04:16:30.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:16:30 smithi195 ceph-mon[49435]: from='client.34234 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T04:16:30.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:16:30 smithi195 ceph-mon[49435]: from='client.34238 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T04:16:30.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:16:30 smithi195 ceph-mon[49435]: osd.1 marked itself down and dead 2024-09-21T04:16:30.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:16:30 smithi195 ceph-mon[49435]: from='client.? 172.21.15.89:0/1809051875' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T04:16:30.365 INFO:teuthology.orchestra.run.smithi089.stdout: 5 180900 10.15 MB/sec execute 309 sec latency 213.497 ms 2024-09-21T04:16:30.552 INFO:teuthology.orchestra.run.smithi089.stdout:HEALTH_WARN 1 osds down 2024-09-21T04:16:30.552 INFO:teuthology.orchestra.run.smithi089.stdout:[WRN] OSD_DOWN: 1 osds down 2024-09-21T04:16:30.552 INFO:teuthology.orchestra.run.smithi089.stdout: osd.1 (root=default,host=smithi089) is down 2024-09-21T04:16:30.867 INFO:journalctl@ceph.osd.1.smithi089.stdout:Sep 21 04:16:30 smithi089 podman[70877]: 2024-09-21 04:16:30.799427111 +0000 UTC m=+1.606833490 container died f9ec697d08d083465ce56c4b414a272680a621f6a3b8d966bbeec3c8d5cd3ee1 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-1, vendor=Red Hat, Inc., name=centos-stream, io.openshift.tags=base centos centos-stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, build-date=2022-05-03T08:36:31.336870, ceph=True, GIT_CLEAN=True, vcs-type=git, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.k8s.display-name=CentOS Stream 8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, release=754, version=8, io.buildah.version=1.19.8, architecture=x86_64, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_BRANCH=HEAD, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.component=centos-stream-container, CEPH_POINT_RELEASE=-17.2.0, distribution-scope=public, io.openshift.expose-services=, RELEASE=HEAD, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, maintainer=Guillaume Abrioux ) 2024-09-21T04:16:31.134 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:16:31 smithi089 ceph-mon[64076]: pgmap v86: 97 pgs: 97 active+clean; 321 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 5.7 MiB/s wr, 557 op/s 2024-09-21T04:16:31.134 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:16:31 smithi089 ceph-mon[64076]: from='client.34246 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T04:16:31.134 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:16:31 smithi089 ceph-mon[64076]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-21T04:16:31.134 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:16:31 smithi089 ceph-mon[64076]: osdmap e84: 8 total, 7 up, 8 in 2024-09-21T04:16:31.134 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:16:31 smithi089 ceph-mon[64076]: from='client.? 172.21.15.89:0/2922307195' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-21T04:16:31.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:16:31 smithi195 ceph-mon[49435]: pgmap v86: 97 pgs: 97 active+clean; 321 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 5.7 MiB/s wr, 557 op/s 2024-09-21T04:16:31.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:16:31 smithi195 ceph-mon[49435]: from='client.34246 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T04:16:31.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:16:31 smithi195 ceph-mon[49435]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-21T04:16:31.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:16:31 smithi195 ceph-mon[49435]: osdmap e84: 8 total, 7 up, 8 in 2024-09-21T04:16:31.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:16:31 smithi195 ceph-mon[49435]: from='client.? 172.21.15.89:0/2922307195' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-21T04:16:31.366 INFO:teuthology.orchestra.run.smithi089.stdout: 5 181079 10.13 MB/sec execute 310 sec latency 970.264 ms 2024-09-21T04:16:31.641 INFO:journalctl@ceph.osd.1.smithi089.stdout:Sep 21 04:16:31 smithi089 podman[70877]: 2024-09-21 04:16:31.388387003 +0000 UTC m=+2.195793386 container cleanup f9ec697d08d083465ce56c4b414a272680a621f6a3b8d966bbeec3c8d5cd3ee1 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-1, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, release=754, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vcs-type=git, distribution-scope=public, io.openshift.expose-services=, maintainer=Guillaume Abrioux , GIT_REPO=https://github.com/ceph/ceph-container.git, build-date=2022-05-03T08:36:31.336870, ceph=True, io.openshift.tags=base centos centos-stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, CEPH_POINT_RELEASE=-17.2.0, RELEASE=HEAD, GIT_BRANCH=HEAD, vendor=Red Hat, Inc., io.k8s.display-name=CentOS Stream 8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, com.redhat.component=centos-stream-container, GIT_CLEAN=True, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.buildah.version=1.19.8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., version=8, name=centos-stream, architecture=x86_64, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754) 2024-09-21T04:16:31.641 INFO:journalctl@ceph.osd.1.smithi089.stdout:Sep 21 04:16:31 smithi089 bash[70877]: ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-1 2024-09-21T04:16:31.913 INFO:journalctl@ceph.osd.1.smithi089.stdout:Sep 21 04:16:31 smithi089 podman[70942]: 2024-09-21 04:16:31.697602977 +0000 UTC m=+0.890904487 container remove f9ec697d08d083465ce56c4b414a272680a621f6a3b8d966bbeec3c8d5cd3ee1 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-1, RELEASE=HEAD, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vendor=Red Hat, Inc., GIT_BRANCH=HEAD, version=8, CEPH_POINT_RELEASE=-17.2.0, vcs-type=git, GIT_CLEAN=True, architecture=x86_64, maintainer=Guillaume Abrioux , description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, build-date=2022-05-03T08:36:31.336870, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_REPO=https://github.com/ceph/ceph-container.git, distribution-scope=public, io.k8s.display-name=CentOS Stream 8, com.redhat.component=centos-stream-container, io.buildah.version=1.19.8, release=754, name=centos-stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.openshift.expose-services=, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.license_terms=https://centos.org/legal/licensing-policy/, ceph=True, io.openshift.tags=base centos centos-stream) 2024-09-21T04:16:32.164 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:16:32 smithi089 ceph-mon[64076]: osdmap e85: 8 total, 7 up, 8 in 2024-09-21T04:16:32.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:16:32 smithi195 ceph-mon[49435]: osdmap e85: 8 total, 7 up, 8 in 2024-09-21T04:16:32.365 INFO:teuthology.orchestra.run.smithi089.stdout: 5 181319 10.12 MB/sec execute 311 sec latency 86.609 ms 2024-09-21T04:16:32.420 INFO:journalctl@ceph.osd.1.smithi089.stdout:Sep 21 04:16:32 smithi089 podman[71027]: 2024-09-21 04:16:32.153762588 +0000 UTC m=+0.019514399 image pull 48ed0a2cf89d6d0371a2c6185a5869a747265f6abd5ac2a74f00cc32d5628721 quay.ceph.io/ceph-ci/ceph@sha256:2195e5f178be0a9390728e2a14088236d8fe6f82635f5c84418fe37a04457e00 2024-09-21T04:16:32.421 INFO:journalctl@ceph.osd.1.smithi089.stdout:Sep 21 04:16:32 smithi089 podman[71027]: 2024-09-21 04:16:32.345792407 +0000 UTC m=+0.211544212 container create f88c674b0b4319638d5d52e9eab4c06e5dc7bef95f3daab5afce477fe281082a (image=quay.ceph.io/ceph-ci/ceph@sha256:2195e5f178be0a9390728e2a14088236d8fe6f82635f5c84418fe37a04457e00, name=ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-1-deactivate, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.vendor=CentOS, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, RELEASE=wip-guits-main-2024-09-20-2310-f36283e) 2024-09-21T04:16:32.941 INFO:journalctl@ceph.osd.1.smithi089.stdout:Sep 21 04:16:32 smithi089 podman[71027]: 2024-09-21 04:16:32.724166527 +0000 UTC m=+0.589918334 container init f88c674b0b4319638d5d52e9eab4c06e5dc7bef95f3daab5afce477fe281082a (image=quay.ceph.io/ceph-ci/ceph@sha256:2195e5f178be0a9390728e2a14088236d8fe6f82635f5c84418fe37a04457e00, name=ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-1-deactivate, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=wip-guits-main-2024-09-20-2310-f36283e, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, ceph=True, CEPH_POINT_RELEASE=, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, GIT_BRANCH=HEAD) 2024-09-21T04:16:32.941 INFO:journalctl@ceph.osd.1.smithi089.stdout:Sep 21 04:16:32 smithi089 podman[71027]: 2024-09-21 04:16:32.730174635 +0000 UTC m=+0.595926442 container start f88c674b0b4319638d5d52e9eab4c06e5dc7bef95f3daab5afce477fe281082a (image=quay.ceph.io/ceph-ci/ceph@sha256:2195e5f178be0a9390728e2a14088236d8fe6f82635f5c84418fe37a04457e00, name=ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-1-deactivate, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_BRANCH=HEAD, ceph=True, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, RELEASE=wip-guits-main-2024-09-20-2310-f36283e) 2024-09-21T04:16:32.941 INFO:journalctl@ceph.osd.1.smithi089.stdout:Sep 21 04:16:32 smithi089 podman[71027]: 2024-09-21 04:16:32.847868061 +0000 UTC m=+0.713619888 container attach f88c674b0b4319638d5d52e9eab4c06e5dc7bef95f3daab5afce477fe281082a (image=quay.ceph.io/ceph-ci/ceph@sha256:2195e5f178be0a9390728e2a14088236d8fe6f82635f5c84418fe37a04457e00, name=ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-1-deactivate, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , ceph=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, RELEASE=wip-guits-main-2024-09-20-2310-f36283e, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_CLEAN=True) 2024-09-21T04:16:33.201 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:16:33 smithi089 ceph-mon[64076]: pgmap v89: 97 pgs: 14 stale+active+clean, 83 active+clean; 321 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 5.6 KiB/s rd, 2.7 MiB/s wr, 313 op/s 2024-09-21T04:16:33.201 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:16:33 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:16:33.201 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:16:33 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T04:16:33.201 INFO:journalctl@ceph.osd.1.smithi089.stdout:Sep 21 04:16:33 smithi089 podman[71027]: 2024-09-21 04:16:33.046997712 +0000 UTC m=+0.912749524 container died f88c674b0b4319638d5d52e9eab4c06e5dc7bef95f3daab5afce477fe281082a (image=quay.ceph.io/ceph-ci/ceph@sha256:2195e5f178be0a9390728e2a14088236d8fe6f82635f5c84418fe37a04457e00, name=ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-1-deactivate, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, RELEASE=wip-guits-main-2024-09-20-2310-f36283e, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , ceph=True, GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23) 2024-09-21T04:16:33.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:16:33 smithi195 ceph-mon[49435]: pgmap v89: 97 pgs: 14 stale+active+clean, 83 active+clean; 321 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 5.6 KiB/s rd, 2.7 MiB/s wr, 313 op/s 2024-09-21T04:16:33.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:16:33 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:16:33.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:16:33 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T04:16:33.366 INFO:teuthology.orchestra.run.smithi089.stdout: 5 181536 10.11 MB/sec execute 312 sec latency 59.593 ms 2024-09-21T04:16:34.226 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:16:34 smithi089 ceph-mon[64076]: Health check failed: Degraded data redundancy: 176/1632 objects degraded (10.784%), 23 pgs degraded (PG_DEGRADED) 2024-09-21T04:16:34.366 INFO:teuthology.orchestra.run.smithi089.stdout: 5 181746 10.08 MB/sec execute 313 sec latency 110.566 ms 2024-09-21T04:16:34.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:16:34 smithi195 ceph-mon[49435]: Health check failed: Degraded data redundancy: 176/1632 objects degraded (10.784%), 23 pgs degraded (PG_DEGRADED) 2024-09-21T04:16:34.734 INFO:journalctl@ceph.osd.1.smithi089.stdout:Sep 21 04:16:34 smithi089 podman[71065]: 2024-09-21 04:16:34.648324096 +0000 UTC m=+1.588970019 container remove f88c674b0b4319638d5d52e9eab4c06e5dc7bef95f3daab5afce477fe281082a (image=quay.ceph.io/ceph-ci/ceph@sha256:2195e5f178be0a9390728e2a14088236d8fe6f82635f5c84418fe37a04457e00, name=ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-1-deactivate, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, org.label-schema.vendor=CentOS, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, RELEASE=wip-guits-main-2024-09-20-2310-f36283e) 2024-09-21T04:16:34.734 INFO:journalctl@ceph.osd.1.smithi089.stdout:Sep 21 04:16:34 smithi089 systemd[1]: ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490@osd.1.service: Deactivated successfully. 2024-09-21T04:16:34.734 INFO:journalctl@ceph.osd.1.smithi089.stdout:Sep 21 04:16:34 smithi089 systemd[1]: Stopped Ceph osd.1 for 1b9b79fc-77ce-11ef-baf5-efdc52797490. 2024-09-21T04:16:34.734 INFO:journalctl@ceph.osd.1.smithi089.stdout:Sep 21 04:16:34 smithi089 systemd[1]: ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490@osd.1.service: Consumed 3min 3.734s CPU time. 2024-09-21T04:16:35.252 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:16:35 smithi089 ceph-mon[64076]: pgmap v90: 97 pgs: 16 active+undersized, 23 active+undersized+degraded, 58 active+clean; 327 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 8.9 KiB/s rd, 5.8 MiB/s wr, 566 op/s; 176/1632 objects degraded (10.784%) 2024-09-21T04:16:35.252 INFO:journalctl@ceph.osd.1.smithi089.stdout:Sep 21 04:16:35 smithi089 systemd[1]: Starting Ceph osd.1 for 1b9b79fc-77ce-11ef-baf5-efdc52797490... 2024-09-21T04:16:35.366 INFO:teuthology.orchestra.run.smithi089.stdout: 5 182042 10.06 MB/sec execute 314 sec latency 62.974 ms 2024-09-21T04:16:35.513 INFO:journalctl@ceph.osd.1.smithi089.stdout:Sep 21 04:16:35 smithi089 podman[71162]: 2024-09-21 04:16:35.170879492 +0000 UTC m=+0.018397261 image pull 48ed0a2cf89d6d0371a2c6185a5869a747265f6abd5ac2a74f00cc32d5628721 quay.ceph.io/ceph-ci/ceph@sha256:2195e5f178be0a9390728e2a14088236d8fe6f82635f5c84418fe37a04457e00 2024-09-21T04:16:35.513 INFO:journalctl@ceph.osd.1.smithi089.stdout:Sep 21 04:16:35 smithi089 podman[71162]: 2024-09-21 04:16:35.301862634 +0000 UTC m=+0.149380381 container create 49cd6c460101b84fbdfc2d64a82b323624d8e58e142d26b63a4f699b223cf56d (image=quay.ceph.io/ceph-ci/ceph@sha256:2195e5f178be0a9390728e2a14088236d8fe6f82635f5c84418fe37a04457e00, name=ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-1-activate, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=wip-guits-main-2024-09-20-2310-f36283e, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23) 2024-09-21T04:16:35.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:16:35 smithi195 ceph-mon[49435]: pgmap v90: 97 pgs: 16 active+undersized, 23 active+undersized+degraded, 58 active+clean; 327 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 8.9 KiB/s rd, 5.8 MiB/s wr, 566 op/s; 176/1632 objects degraded (10.784%) 2024-09-21T04:16:35.764 INFO:journalctl@ceph.osd.1.smithi089.stdout:Sep 21 04:16:35 smithi089 podman[71162]: 2024-09-21 04:16:35.654328922 +0000 UTC m=+0.501846669 container init 49cd6c460101b84fbdfc2d64a82b323624d8e58e142d26b63a4f699b223cf56d (image=quay.ceph.io/ceph-ci/ceph@sha256:2195e5f178be0a9390728e2a14088236d8fe6f82635f5c84418fe37a04457e00, name=ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-1-activate, RELEASE=wip-guits-main-2024-09-20-2310-f36283e, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, io.buildah.version=1.37.2) 2024-09-21T04:16:35.764 INFO:journalctl@ceph.osd.1.smithi089.stdout:Sep 21 04:16:35 smithi089 podman[71162]: 2024-09-21 04:16:35.665284054 +0000 UTC m=+0.512801806 container start 49cd6c460101b84fbdfc2d64a82b323624d8e58e142d26b63a4f699b223cf56d (image=quay.ceph.io/ceph-ci/ceph@sha256:2195e5f178be0a9390728e2a14088236d8fe6f82635f5c84418fe37a04457e00, name=ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-1-activate, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_CLEAN=True, RELEASE=wip-guits-main-2024-09-20-2310-f36283e, ceph=True, org.label-schema.license=GPLv2, io.buildah.version=1.37.2) 2024-09-21T04:16:35.765 INFO:journalctl@ceph.osd.1.smithi089.stdout:Sep 21 04:16:35 smithi089 podman[71162]: 2024-09-21 04:16:35.762841023 +0000 UTC m=+0.610358776 container attach 49cd6c460101b84fbdfc2d64a82b323624d8e58e142d26b63a4f699b223cf56d (image=quay.ceph.io/ceph-ci/ceph@sha256:2195e5f178be0a9390728e2a14088236d8fe6f82635f5c84418fe37a04457e00, name=ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-1-activate, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, GIT_CLEAN=True, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-guits-main-2024-09-20-2310-f36283e) 2024-09-21T04:16:36.014 INFO:journalctl@ceph.osd.1.smithi089.stdout:Sep 21 04:16:35 smithi089 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-1-activate[71177]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T04:16:36.014 INFO:journalctl@ceph.osd.1.smithi089.stdout:Sep 21 04:16:35 smithi089 bash[71162]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T04:16:36.015 INFO:journalctl@ceph.osd.1.smithi089.stdout:Sep 21 04:16:35 smithi089 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-1-activate[71177]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T04:16:36.015 INFO:journalctl@ceph.osd.1.smithi089.stdout:Sep 21 04:16:35 smithi089 bash[71162]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T04:16:36.366 INFO:teuthology.orchestra.run.smithi089.stdout: 5 182223 10.04 MB/sec execute 315 sec latency 64.515 ms 2024-09-21T04:16:37.311 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:16:37 smithi089 ceph-mon[64076]: pgmap v91: 97 pgs: 16 active+undersized, 23 active+undersized+degraded, 58 active+clean; 327 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 8.9 KiB/s rd, 4.7 MiB/s wr, 440 op/s; 176/1632 objects degraded (10.784%) 2024-09-21T04:16:37.366 INFO:teuthology.orchestra.run.smithi089.stdout: 5 182470 10.04 MB/sec execute 316 sec latency 68.111 ms 2024-09-21T04:16:37.411 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:16:37 smithi195 ceph-mon[49435]: pgmap v91: 97 pgs: 16 active+undersized, 23 active+undersized+degraded, 58 active+clean; 327 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 8.9 KiB/s rd, 4.7 MiB/s wr, 440 op/s; 176/1632 objects degraded (10.784%) 2024-09-21T04:16:38.354 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:16:38 smithi089 ceph-mon[64076]: pgmap v92: 97 pgs: 16 active+undersized, 23 active+undersized+degraded, 58 active+clean; 326 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 3.2 KiB/s rd, 3.7 MiB/s wr, 314 op/s; 177/1605 objects degraded (11.028%) 2024-09-21T04:16:38.366 INFO:teuthology.orchestra.run.smithi089.stdout: 5 182846 10.04 MB/sec execute 317 sec latency 107.609 ms 2024-09-21T04:16:38.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:16:38 smithi195 ceph-mon[49435]: pgmap v92: 97 pgs: 16 active+undersized, 23 active+undersized+degraded, 58 active+clean; 326 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 3.2 KiB/s rd, 3.7 MiB/s wr, 314 op/s; 177/1605 objects degraded (11.028%) 2024-09-21T04:16:39.366 INFO:teuthology.orchestra.run.smithi089.stdout: 5 183276 10.03 MB/sec execute 318 sec latency 52.067 ms 2024-09-21T04:16:40.139 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:16:39 smithi089 ceph-mon[64076]: Health check update: Degraded data redundancy: 177/1605 objects degraded (11.028%), 23 pgs degraded (PG_DEGRADED) 2024-09-21T04:16:40.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:16:39 smithi195 ceph-mon[49435]: Health check update: Degraded data redundancy: 177/1605 objects degraded (11.028%), 23 pgs degraded (PG_DEGRADED) 2024-09-21T04:16:40.366 INFO:teuthology.orchestra.run.smithi089.stdout: 5 183610 10.03 MB/sec execute 319 sec latency 61.928 ms 2024-09-21T04:16:40.393 INFO:journalctl@ceph.osd.1.smithi089.stdout:Sep 21 04:16:40 smithi089 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-1-activate[71177]: --> Failed to activate via raw: did not find any matching OSD to activate 2024-09-21T04:16:40.393 INFO:journalctl@ceph.osd.1.smithi089.stdout:Sep 21 04:16:40 smithi089 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-1-activate[71177]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T04:16:40.393 INFO:journalctl@ceph.osd.1.smithi089.stdout:Sep 21 04:16:40 smithi089 bash[71162]: --> Failed to activate via raw: did not find any matching OSD to activate 2024-09-21T04:16:40.393 INFO:journalctl@ceph.osd.1.smithi089.stdout:Sep 21 04:16:40 smithi089 bash[71162]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T04:16:40.393 INFO:journalctl@ceph.osd.1.smithi089.stdout:Sep 21 04:16:40 smithi089 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-1-activate[71177]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T04:16:40.393 INFO:journalctl@ceph.osd.1.smithi089.stdout:Sep 21 04:16:40 smithi089 bash[71162]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T04:16:40.646 INFO:journalctl@ceph.osd.1.smithi089.stdout:Sep 21 04:16:40 smithi089 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-1-activate[71177]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-09-21T04:16:40.646 INFO:journalctl@ceph.osd.1.smithi089.stdout:Sep 21 04:16:40 smithi089 bash[71162]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-09-21T04:16:40.646 INFO:journalctl@ceph.osd.1.smithi089.stdout:Sep 21 04:16:40 smithi089 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-1-activate[71177]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_3 --path /var/lib/ceph/osd/ceph-1 --no-mon-config 2024-09-21T04:16:40.646 INFO:journalctl@ceph.osd.1.smithi089.stdout:Sep 21 04:16:40 smithi089 bash[71162]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_3 --path /var/lib/ceph/osd/ceph-1 --no-mon-config 2024-09-21T04:16:40.899 INFO:journalctl@ceph.osd.1.smithi089.stdout:Sep 21 04:16:40 smithi089 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-1-activate[71177]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_3 /var/lib/ceph/osd/ceph-1/block 2024-09-21T04:16:40.899 INFO:journalctl@ceph.osd.1.smithi089.stdout:Sep 21 04:16:40 smithi089 bash[71162]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_3 /var/lib/ceph/osd/ceph-1/block 2024-09-21T04:16:40.899 INFO:journalctl@ceph.osd.1.smithi089.stdout:Sep 21 04:16:40 smithi089 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-1-activate[71177]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-1/block 2024-09-21T04:16:40.899 INFO:journalctl@ceph.osd.1.smithi089.stdout:Sep 21 04:16:40 smithi089 bash[71162]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-1/block 2024-09-21T04:16:40.900 INFO:journalctl@ceph.osd.1.smithi089.stdout:Sep 21 04:16:40 smithi089 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-1-activate[71177]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-09-21T04:16:40.900 INFO:journalctl@ceph.osd.1.smithi089.stdout:Sep 21 04:16:40 smithi089 bash[71162]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-09-21T04:16:40.900 INFO:journalctl@ceph.osd.1.smithi089.stdout:Sep 21 04:16:40 smithi089 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-1-activate[71177]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-09-21T04:16:40.900 INFO:journalctl@ceph.osd.1.smithi089.stdout:Sep 21 04:16:40 smithi089 bash[71162]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-09-21T04:16:40.900 INFO:journalctl@ceph.osd.1.smithi089.stdout:Sep 21 04:16:40 smithi089 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-1-activate[71177]: --> ceph-volume lvm activate successful for osd ID: 1 2024-09-21T04:16:40.900 INFO:journalctl@ceph.osd.1.smithi089.stdout:Sep 21 04:16:40 smithi089 bash[71162]: --> ceph-volume lvm activate successful for osd ID: 1 2024-09-21T04:16:41.155 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:16:40 smithi089 ceph-mon[64076]: pgmap v93: 97 pgs: 16 active+undersized, 23 active+undersized+degraded, 58 active+clean; 323 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 7.0 KiB/s rd, 4.0 MiB/s wr, 405 op/s; 171/1494 objects degraded (11.446%) 2024-09-21T04:16:41.156 INFO:journalctl@ceph.osd.1.smithi089.stdout:Sep 21 04:16:40 smithi089 podman[71162]: 2024-09-21 04:16:40.934649859 +0000 UTC m=+5.782167618 container died 49cd6c460101b84fbdfc2d64a82b323624d8e58e142d26b63a4f699b223cf56d (image=quay.ceph.io/ceph-ci/ceph@sha256:2195e5f178be0a9390728e2a14088236d8fe6f82635f5c84418fe37a04457e00, name=ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-1-activate, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, RELEASE=wip-guits-main-2024-09-20-2310-f36283e, GIT_BRANCH=HEAD) 2024-09-21T04:16:41.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:16:40 smithi195 ceph-mon[49435]: pgmap v93: 97 pgs: 16 active+undersized, 23 active+undersized+degraded, 58 active+clean; 323 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 7.0 KiB/s rd, 4.0 MiB/s wr, 405 op/s; 171/1494 objects degraded (11.446%) 2024-09-21T04:16:41.366 INFO:teuthology.orchestra.run.smithi089.stdout: 5 183917 10.03 MB/sec execute 320 sec latency 58.105 ms 2024-09-21T04:16:42.366 INFO:teuthology.orchestra.run.smithi089.stdout: 5 184311 10.05 MB/sec execute 321 sec latency 55.728 ms 2024-09-21T04:16:42.449 INFO:journalctl@ceph.osd.1.smithi089.stdout:Sep 21 04:16:42 smithi089 podman[71460]: 2024-09-21 04:16:42.121357803 +0000 UTC m=+0.023211426 image pull 48ed0a2cf89d6d0371a2c6185a5869a747265f6abd5ac2a74f00cc32d5628721 quay.ceph.io/ceph-ci/ceph@sha256:2195e5f178be0a9390728e2a14088236d8fe6f82635f5c84418fe37a04457e00 2024-09-21T04:16:42.449 INFO:journalctl@ceph.osd.1.smithi089.stdout:Sep 21 04:16:42 smithi089 podman[71460]: 2024-09-21 04:16:42.260472983 +0000 UTC m=+0.162326604 container create da2e49774e7ec1d3618c3ed46871ffb1cd2bf3b990986548c2d891e2c3054e66 (image=quay.ceph.io/ceph-ci/ceph@sha256:2195e5f178be0a9390728e2a14088236d8fe6f82635f5c84418fe37a04457e00, name=ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-1, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, RELEASE=wip-guits-main-2024-09-20-2310-f36283e, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=) 2024-09-21T04:16:42.702 INFO:journalctl@ceph.osd.1.smithi089.stdout:Sep 21 04:16:42 smithi089 podman[71460]: 2024-09-21 04:16:42.655263245 +0000 UTC m=+0.557116880 container init da2e49774e7ec1d3618c3ed46871ffb1cd2bf3b990986548c2d891e2c3054e66 (image=quay.ceph.io/ceph-ci/ceph@sha256:2195e5f178be0a9390728e2a14088236d8fe6f82635f5c84418fe37a04457e00, name=ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-1, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_BRANCH=HEAD, ceph=True, maintainer=Guillaume Abrioux , GIT_CLEAN=True, RELEASE=wip-guits-main-2024-09-20-2310-f36283e, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-21T04:16:42.702 INFO:journalctl@ceph.osd.1.smithi089.stdout:Sep 21 04:16:42 smithi089 podman[71460]: 2024-09-21 04:16:42.668309184 +0000 UTC m=+0.570162804 container start da2e49774e7ec1d3618c3ed46871ffb1cd2bf3b990986548c2d891e2c3054e66 (image=quay.ceph.io/ceph-ci/ceph@sha256:2195e5f178be0a9390728e2a14088236d8fe6f82635f5c84418fe37a04457e00, name=ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-1, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_CLEAN=True, RELEASE=wip-guits-main-2024-09-20-2310-f36283e, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, org.label-schema.license=GPLv2) 2024-09-21T04:16:42.957 INFO:journalctl@ceph.osd.1.smithi089.stdout:Sep 21 04:16:42 smithi089 ceph-osd[71478]: -- 172.21.15.89:0/2598961229 <== mon.0 v2:172.21.15.89:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x5588fc9701e0 con 0x5588fc904000 2024-09-21T04:16:42.957 INFO:journalctl@ceph.osd.1.smithi089.stdout:Sep 21 04:16:42 smithi089 bash[71460]: da2e49774e7ec1d3618c3ed46871ffb1cd2bf3b990986548c2d891e2c3054e66 2024-09-21T04:16:42.957 INFO:journalctl@ceph.osd.1.smithi089.stdout:Sep 21 04:16:42 smithi089 systemd[1]: Started Ceph osd.1 for 1b9b79fc-77ce-11ef-baf5-efdc52797490. 2024-09-21T04:16:43.226 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:16:43 smithi089 ceph-mon[64076]: pgmap v94: 97 pgs: 16 active+undersized, 23 active+undersized+degraded, 58 active+clean; 323 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 6.3 KiB/s rd, 3.7 MiB/s wr, 369 op/s; 171/1494 objects degraded (11.446%) 2024-09-21T04:16:43.226 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:16:43 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:16:43.226 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:16:43 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:16:43.226 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:16:43 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T04:16:43.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:16:43 smithi195 ceph-mon[49435]: pgmap v94: 97 pgs: 16 active+undersized, 23 active+undersized+degraded, 58 active+clean; 323 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 6.3 KiB/s rd, 3.7 MiB/s wr, 369 op/s; 171/1494 objects degraded (11.446%) 2024-09-21T04:16:43.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:16:43 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:16:43.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:16:43 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:16:43.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:16:43 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T04:16:43.366 INFO:teuthology.orchestra.run.smithi089.stdout: 5 184630 10.05 MB/sec execute 322 sec latency 50.641 ms 2024-09-21T04:16:44.366 INFO:teuthology.orchestra.run.smithi089.stdout: 5 184889 10.05 MB/sec execute 323 sec latency 40.466 ms 2024-09-21T04:16:44.748 INFO:journalctl@ceph.osd.1.smithi089.stdout:Sep 21 04:16:44 smithi089 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-1[71474]: 2024-09-21T04:16:44.730+0000 7f4ca27b9640 -1 Falling back to public interface 2024-09-21T04:16:45.265 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:16:45 smithi089 ceph-mon[64076]: pgmap v95: 97 pgs: 16 active+undersized, 23 active+undersized+degraded, 58 active+clean; 327 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 7.7 KiB/s rd, 5.6 MiB/s wr, 542 op/s; 202/1674 objects degraded (12.067%) 2024-09-21T04:16:45.265 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:16:45 smithi089 ceph-mon[64076]: Health check update: Degraded data redundancy: 202/1674 objects degraded (12.067%), 23 pgs degraded (PG_DEGRADED) 2024-09-21T04:16:45.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:16:45 smithi195 ceph-mon[49435]: pgmap v95: 97 pgs: 16 active+undersized, 23 active+undersized+degraded, 58 active+clean; 327 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 7.7 KiB/s rd, 5.6 MiB/s wr, 542 op/s; 202/1674 objects degraded (12.067%) 2024-09-21T04:16:45.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:16:45 smithi195 ceph-mon[49435]: Health check update: Degraded data redundancy: 202/1674 objects degraded (12.067%), 23 pgs degraded (PG_DEGRADED) 2024-09-21T04:16:45.366 INFO:teuthology.orchestra.run.smithi089.stdout: 5 185113 10.03 MB/sec execute 324 sec latency 38.061 ms 2024-09-21T04:16:46.366 INFO:teuthology.orchestra.run.smithi089.stdout: 5 185395 10.01 MB/sec execute 325 sec latency 67.007 ms 2024-09-21T04:16:47.303 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:16:47 smithi089 ceph-mon[64076]: pgmap v96: 97 pgs: 16 active+undersized, 23 active+undersized+degraded, 58 active+clean; 327 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 5.6 KiB/s rd, 3.6 MiB/s wr, 373 op/s; 202/1674 objects degraded (12.067%) 2024-09-21T04:16:47.358 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:16:47 smithi195 ceph-mon[49435]: pgmap v96: 97 pgs: 16 active+undersized, 23 active+undersized+degraded, 58 active+clean; 327 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 5.6 KiB/s rd, 3.6 MiB/s wr, 373 op/s; 202/1674 objects degraded (12.067%) 2024-09-21T04:16:47.366 INFO:teuthology.orchestra.run.smithi089.stdout: 5 185678 9.99 MB/sec execute 326 sec latency 69.171 ms 2024-09-21T04:16:48.366 INFO:teuthology.orchestra.run.smithi089.stdout: 5 185965 9.99 MB/sec execute 327 sec latency 55.070 ms 2024-09-21T04:16:48.579 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:16:48 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:16:48.579 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:16:48 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T04:16:48.579 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:16:48 smithi089 ceph-mon[64076]: pgmap v97: 97 pgs: 16 active+undersized, 23 active+undersized+degraded, 58 active+clean; 329 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 5.6 KiB/s rd, 4.4 MiB/s wr, 465 op/s; 195/1644 objects degraded (11.861%) 2024-09-21T04:16:48.859 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:16:48 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:16:48.859 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:16:48 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T04:16:48.859 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:16:48 smithi195 ceph-mon[49435]: pgmap v97: 97 pgs: 16 active+undersized, 23 active+undersized+degraded, 58 active+clean; 329 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 5.6 KiB/s rd, 4.4 MiB/s wr, 465 op/s; 195/1644 objects degraded (11.861%) 2024-09-21T04:16:49.367 INFO:teuthology.orchestra.run.smithi089.stdout: 5 186441 10.00 MB/sec execute 328 sec latency 41.730 ms 2024-09-21T04:16:50.101 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:16:49 smithi089 ceph-mon[64076]: Health check update: Degraded data redundancy: 195/1644 objects degraded (11.861%), 23 pgs degraded (PG_DEGRADED) 2024-09-21T04:16:50.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:16:49 smithi195 ceph-mon[49435]: Health check update: Degraded data redundancy: 195/1644 objects degraded (11.861%), 23 pgs degraded (PG_DEGRADED) 2024-09-21T04:16:50.366 INFO:teuthology.orchestra.run.smithi089.stdout: 5 186827 9.99 MB/sec execute 329 sec latency 48.027 ms 2024-09-21T04:16:51.124 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:16:50 smithi089 ceph-mon[64076]: pgmap v98: 97 pgs: 16 active+undersized, 23 active+undersized+degraded, 58 active+clean; 322 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 9.2 KiB/s rd, 4.9 MiB/s wr, 514 op/s; 172/1554 objects degraded (11.068%) 2024-09-21T04:16:51.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:16:50 smithi195 ceph-mon[49435]: pgmap v98: 97 pgs: 16 active+undersized, 23 active+undersized+degraded, 58 active+clean; 322 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 9.2 KiB/s rd, 4.9 MiB/s wr, 514 op/s; 172/1554 objects degraded (11.068%) 2024-09-21T04:16:51.366 INFO:teuthology.orchestra.run.smithi089.stdout: 5 187128 9.98 MB/sec execute 330 sec latency 94.431 ms 2024-09-21T04:16:52.367 INFO:teuthology.orchestra.run.smithi089.stdout: 5 187437 9.99 MB/sec execute 331 sec latency 65.717 ms 2024-09-21T04:16:52.400 INFO:journalctl@ceph.osd.1.smithi089.stdout:Sep 21 04:16:52 smithi089 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-1[71474]: 2024-09-21T04:16:52.346+0000 7f4ca27b9640 -1 osd.1 0 read_superblock omap replica is missing. 2024-09-21T04:16:52.655 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:16:52 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:16:52.655 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:16:52 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:16:52.655 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:16:52 smithi089 ceph-mon[64076]: pgmap v99: 97 pgs: 16 active+undersized, 23 active+undersized+degraded, 58 active+clean; 322 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 5.6 KiB/s rd, 4.0 MiB/s wr, 389 op/s; 172/1554 objects degraded (11.068%) 2024-09-21T04:16:52.655 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:16:52 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:16:52.655 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:16:52 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:16:52.859 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:16:52 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:16:52.859 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:16:52 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:16:52.859 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:16:52 smithi195 ceph-mon[49435]: pgmap v99: 97 pgs: 16 active+undersized, 23 active+undersized+degraded, 58 active+clean; 322 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 5.6 KiB/s rd, 4.0 MiB/s wr, 389 op/s; 172/1554 objects degraded (11.068%) 2024-09-21T04:16:52.859 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:16:52 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:16:52.859 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:16:52 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:16:53.367 INFO:teuthology.orchestra.run.smithi089.stdout: 5 187913 10.02 MB/sec execute 332 sec latency 34.519 ms 2024-09-21T04:16:53.685 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:16:53 smithi089 ceph-mon[64076]: from='osd.1 [v2:172.21.15.89:6810/1072208504,v1:172.21.15.89:6811/1072208504]' entity='osd.1' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]} : dispatch 2024-09-21T04:16:53.685 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:16:53 smithi089 ceph-mon[64076]: from='osd.1 ' entity='osd.1' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]} : dispatch 2024-09-21T04:16:53.685 INFO:journalctl@ceph.osd.1.smithi089.stdout:Sep 21 04:16:53 smithi089 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-1[71474]: 2024-09-21T04:16:53.485+0000 7f4ca27b9640 -1 osd.1 83 log_to_monitors true 2024-09-21T04:16:53.685 INFO:journalctl@ceph.osd.1.smithi089.stdout:Sep 21 04:16:53 smithi089 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-1[71474]: 2024-09-21T04:16:53.609+0000 7f4c9a563640 -1 osd.1 83 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-21T04:16:53.859 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:16:53 smithi195 ceph-mon[49435]: from='osd.1 [v2:172.21.15.89:6810/1072208504,v1:172.21.15.89:6811/1072208504]' entity='osd.1' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]} : dispatch 2024-09-21T04:16:53.859 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:16:53 smithi195 ceph-mon[49435]: from='osd.1 ' entity='osd.1' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]} : dispatch 2024-09-21T04:16:54.367 INFO:teuthology.orchestra.run.smithi089.stdout: 5 188236 10.01 MB/sec execute 333 sec latency 41.653 ms 2024-09-21T04:16:54.713 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:16:54 smithi089 ceph-mon[64076]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-09-21T04:16:54.713 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:16:54 smithi089 ceph-mon[64076]: osdmap e86: 8 total, 7 up, 8 in 2024-09-21T04:16:54.713 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:16:54 smithi089 ceph-mon[64076]: from='osd.1 [v2:172.21.15.89:6810/1072208504,v1:172.21.15.89:6811/1072208504]' entity='osd.1' cmd={"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi089", "root=default"]} : dispatch 2024-09-21T04:16:54.713 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:16:54 smithi089 ceph-mon[64076]: from='osd.1 ' entity='osd.1' cmd={"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi089", "root=default"]} : dispatch 2024-09-21T04:16:54.713 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:16:54 smithi089 ceph-mon[64076]: pgmap v101: 97 pgs: 16 active+undersized, 23 active+undersized+degraded, 58 active+clean; 329 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 7.3 KiB/s rd, 4.8 MiB/s wr, 481 op/s; 176/1659 objects degraded (10.609%) 2024-09-21T04:16:54.859 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:16:54 smithi195 ceph-mon[49435]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-09-21T04:16:54.859 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:16:54 smithi195 ceph-mon[49435]: osdmap e86: 8 total, 7 up, 8 in 2024-09-21T04:16:54.859 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:16:54 smithi195 ceph-mon[49435]: from='osd.1 [v2:172.21.15.89:6810/1072208504,v1:172.21.15.89:6811/1072208504]' entity='osd.1' cmd={"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi089", "root=default"]} : dispatch 2024-09-21T04:16:54.859 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:16:54 smithi195 ceph-mon[49435]: from='osd.1 ' entity='osd.1' cmd={"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi089", "root=default"]} : dispatch 2024-09-21T04:16:54.859 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:16:54 smithi195 ceph-mon[49435]: pgmap v101: 97 pgs: 16 active+undersized, 23 active+undersized+degraded, 58 active+clean; 329 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 7.3 KiB/s rd, 4.8 MiB/s wr, 481 op/s; 176/1659 objects degraded (10.609%) 2024-09-21T04:16:55.367 INFO:teuthology.orchestra.run.smithi089.stdout: 5 188489 10.01 MB/sec execute 334 sec latency 328.772 ms 2024-09-21T04:16:55.740 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:16:55 smithi089 ceph-mon[64076]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-21T04:16:55.740 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:16:55 smithi089 ceph-mon[64076]: osd.1 [v2:172.21.15.89:6810/1072208504,v1:172.21.15.89:6811/1072208504] boot 2024-09-21T04:16:55.740 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:16:55 smithi089 ceph-mon[64076]: osdmap e87: 8 total, 8 up, 8 in 2024-09-21T04:16:55.740 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:16:55 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "osd metadata", "id": 1} : dispatch 2024-09-21T04:16:55.740 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:16:55 smithi089 ceph-mon[64076]: Health check update: Degraded data redundancy: 176/1659 objects degraded (10.609%), 23 pgs degraded (PG_DEGRADED) 2024-09-21T04:16:55.740 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:16:55 smithi089 ceph-mon[64076]: osdmap e88: 8 total, 8 up, 8 in 2024-09-21T04:16:55.859 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:16:55 smithi195 ceph-mon[49435]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-21T04:16:55.859 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:16:55 smithi195 ceph-mon[49435]: osd.1 [v2:172.21.15.89:6810/1072208504,v1:172.21.15.89:6811/1072208504] boot 2024-09-21T04:16:55.859 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:16:55 smithi195 ceph-mon[49435]: osdmap e87: 8 total, 8 up, 8 in 2024-09-21T04:16:55.859 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:16:55 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "osd metadata", "id": 1} : dispatch 2024-09-21T04:16:55.859 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:16:55 smithi195 ceph-mon[49435]: Health check update: Degraded data redundancy: 176/1659 objects degraded (10.609%), 23 pgs degraded (PG_DEGRADED) 2024-09-21T04:16:55.859 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:16:55 smithi195 ceph-mon[49435]: osdmap e88: 8 total, 8 up, 8 in 2024-09-21T04:16:56.367 INFO:teuthology.orchestra.run.smithi089.stdout: 5 188630 9.99 MB/sec execute 335 sec latency 885.221 ms 2024-09-21T04:16:57.040 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:16:56 smithi089 ceph-mon[64076]: osdmap e89: 8 total, 8 up, 8 in 2024-09-21T04:16:57.040 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:16:56 smithi089 ceph-mon[64076]: pgmap v105: 97 pgs: 16 active+undersized, 23 active+undersized+degraded, 58 active+clean; 329 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 7.5 KiB/s rd, 6.9 MiB/s wr, 659 op/s; 176/1659 objects degraded (10.609%) 2024-09-21T04:16:57.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:16:56 smithi195 ceph-mon[49435]: osdmap e89: 8 total, 8 up, 8 in 2024-09-21T04:16:57.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:16:56 smithi195 ceph-mon[49435]: pgmap v105: 97 pgs: 16 active+undersized, 23 active+undersized+degraded, 58 active+clean; 329 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 7.5 KiB/s rd, 6.9 MiB/s wr, 659 op/s; 176/1659 objects degraded (10.609%) 2024-09-21T04:16:57.367 INFO:teuthology.orchestra.run.smithi089.stdout: 5 188866 9.96 MB/sec execute 336 sec latency 47.630 ms 2024-09-21T04:16:58.078 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:16:57 smithi089 ceph-mon[64076]: osdmap e90: 8 total, 8 up, 8 in 2024-09-21T04:16:58.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:16:57 smithi195 ceph-mon[49435]: osdmap e90: 8 total, 8 up, 8 in 2024-09-21T04:16:58.367 INFO:teuthology.orchestra.run.smithi089.stdout: 5 189011 9.94 MB/sec execute 337 sec latency 378.973 ms 2024-09-21T04:16:59.113 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:16:58 smithi089 ceph-mon[64076]: osdmap e91: 8 total, 8 up, 8 in 2024-09-21T04:16:59.113 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:16:58 smithi089 ceph-mon[64076]: pgmap v108: 97 pgs: 2 peering, 1 active+recovering+undersized+degraded+remapped, 15 active+undersized, 14 active+undersized+degraded, 65 active+clean; 324 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 2.1 MiB/s wr, 226 op/s; 122/1605 objects degraded (7.601%); 0 B/s, 24 objects/s recovering 2024-09-21T04:16:59.113 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:16:58 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:16:59.113 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:16:58 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:16:59.113 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:16:58 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T04:16:59.113 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:16:58 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T04:16:59.114 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:16:58 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:16:59.114 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:16:58 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:16:59.114 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:16:58 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T04:16:59.114 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:16:58 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "versions"} : dispatch 2024-09-21T04:16:59.114 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:16:58 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "versions"} : dispatch 2024-09-21T04:16:59.114 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:16:58 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "versions"} : dispatch 2024-09-21T04:16:59.114 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:16:58 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16} : dispatch 2024-09-21T04:16:59.114 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:16:58 smithi089 ceph-mon[64076]: osdmap e92: 8 total, 8 up, 8 in 2024-09-21T04:16:59.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:16:58 smithi195 ceph-mon[49435]: osdmap e91: 8 total, 8 up, 8 in 2024-09-21T04:16:59.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:16:58 smithi195 ceph-mon[49435]: pgmap v108: 97 pgs: 2 peering, 1 active+recovering+undersized+degraded+remapped, 15 active+undersized, 14 active+undersized+degraded, 65 active+clean; 324 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 2.1 MiB/s wr, 226 op/s; 122/1605 objects degraded (7.601%); 0 B/s, 24 objects/s recovering 2024-09-21T04:16:59.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:16:58 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:16:59.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:16:58 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:16:59.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:16:58 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T04:16:59.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:16:58 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T04:16:59.360 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:16:58 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:16:59.360 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:16:58 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:16:59.360 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:16:58 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T04:16:59.360 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:16:58 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "versions"} : dispatch 2024-09-21T04:16:59.360 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:16:58 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "versions"} : dispatch 2024-09-21T04:16:59.360 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:16:58 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "versions"} : dispatch 2024-09-21T04:16:59.360 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:16:58 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16} : dispatch 2024-09-21T04:16:59.360 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:16:58 smithi195 ceph-mon[49435]: osdmap e92: 8 total, 8 up, 8 in 2024-09-21T04:16:59.367 INFO:teuthology.orchestra.run.smithi089.stdout: 5 189209 9.92 MB/sec execute 338 sec latency 639.202 ms 2024-09-21T04:17:00.141 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:16:59 smithi089 ceph-mon[64076]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-21T04:17:00.141 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:16:59 smithi089 ceph-mon[64076]: Upgrade: unsafe to stop osd(s) at this time (9 PGs are or would become offline) 2024-09-21T04:17:00.141 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:16:59 smithi089 ceph-mon[64076]: Health check update: Degraded data redundancy: 122/1605 objects degraded (7.601%), 15 pgs degraded (PG_DEGRADED) 2024-09-21T04:17:00.141 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:16:59 smithi089 ceph-mon[64076]: osdmap e93: 8 total, 8 up, 8 in 2024-09-21T04:17:00.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:16:59 smithi195 ceph-mon[49435]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-21T04:17:00.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:16:59 smithi195 ceph-mon[49435]: Upgrade: unsafe to stop osd(s) at this time (9 PGs are or would become offline) 2024-09-21T04:17:00.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:16:59 smithi195 ceph-mon[49435]: Health check update: Degraded data redundancy: 122/1605 objects degraded (7.601%), 15 pgs degraded (PG_DEGRADED) 2024-09-21T04:17:00.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:16:59 smithi195 ceph-mon[49435]: osdmap e93: 8 total, 8 up, 8 in 2024-09-21T04:17:00.367 INFO:teuthology.orchestra.run.smithi089.stdout: 5 189495 9.91 MB/sec execute 339 sec latency 72.528 ms 2024-09-21T04:17:00.931 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:17:00 smithi089 ceph-mon[64076]: pgmap v111: 97 pgs: 4 peering, 1 active+recovering+undersized+degraded+remapped, 92 active+clean; 324 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 9.0 KiB/s rd, 4.3 MiB/s wr, 444 op/s; 9/1593 objects degraded (0.565%); 1.3 MiB/s, 70 keys/s, 59 objects/s recovering 2024-09-21T04:17:01.155 INFO:teuthology.orchestra.run.smithi089.stdout:true 2024-09-21T04:17:01.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:17:00 smithi195 ceph-mon[49435]: pgmap v111: 97 pgs: 4 peering, 1 active+recovering+undersized+degraded+remapped, 92 active+clean; 324 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 9.0 KiB/s rd, 4.3 MiB/s wr, 444 op/s; 9/1593 objects degraded (0.565%); 1.3 MiB/s, 70 keys/s, 59 objects/s recovering 2024-09-21T04:17:01.367 INFO:teuthology.orchestra.run.smithi089.stdout: 5 189769 9.90 MB/sec execute 340 sec latency 74.941 ms 2024-09-21T04:17:01.961 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:17:01 smithi089 ceph-mon[64076]: from='client.34258 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T04:17:02.252 INFO:teuthology.orchestra.run.smithi089.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-21T04:17:02.252 INFO:teuthology.orchestra.run.smithi089.stdout:alertmanager.smithi089 smithi089 *:9093,9094 running (5m) 10s ago 14m 17.2M - 0.27.0 11f11916f8cd 65dfcbcafee1 2024-09-21T04:17:02.252 INFO:teuthology.orchestra.run.smithi089.stdout:crash.smithi089 smithi089 running (113s) 10s ago 14m 6891k - 19.3.0-5103-gf36283e7 48ed0a2cf89d 6c415a5e173e 2024-09-21T04:17:02.252 INFO:teuthology.orchestra.run.smithi089.stdout:crash.smithi195 smithi195 running (106s) 100s ago 13m 6891k - 19.3.0-5103-gf36283e7 48ed0a2cf89d 78ce05a4e9ab 2024-09-21T04:17:02.252 INFO:teuthology.orchestra.run.smithi089.stdout:grafana.smithi089 smithi089 *:3000 running (5m) 10s ago 13m 45.4M - dad864ee21e9 ec4d6738bce7 2024-09-21T04:17:02.252 INFO:teuthology.orchestra.run.smithi089.stdout:mds.foofs.smithi089.osufws smithi089 running (8m) 10s ago 8m 38.3M - 17.2.0 e1d6a67b021e 664d41859bea 2024-09-21T04:17:02.252 INFO:teuthology.orchestra.run.smithi089.stdout:mds.foofs.smithi195.hijxre smithi195 running (8m) 100s ago 8m 16.7M - 17.2.0 e1d6a67b021e 7bc0c6542d12 2024-09-21T04:17:02.253 INFO:teuthology.orchestra.run.smithi089.stdout:mgr.smithi089.srlyxf smithi089 *:8443,9283,8765 running (4m) 10s ago 15m 515M - 19.3.0-5103-gf36283e7 48ed0a2cf89d 3cbeaaa88241 2024-09-21T04:17:02.253 INFO:teuthology.orchestra.run.smithi089.stdout:mgr.smithi195.cgotzm smithi195 *:8443,9283,8765 running (3m) 100s ago 13m 454M - 19.3.0-5103-gf36283e7 48ed0a2cf89d d3734574cf66 2024-09-21T04:17:02.253 INFO:teuthology.orchestra.run.smithi089.stdout:mon.smithi089 smithi089 running (3m) 10s ago 15m 49.9M 2048M 19.3.0-5103-gf36283e7 48ed0a2cf89d 7dd34c8a6f21 2024-09-21T04:17:02.253 INFO:teuthology.orchestra.run.smithi089.stdout:mon.smithi195 smithi195 running (2m) 100s ago 13m 36.7M 2048M 19.3.0-5103-gf36283e7 48ed0a2cf89d 4255937444e7 2024-09-21T04:17:02.253 INFO:teuthology.orchestra.run.smithi089.stdout:nfs.foo.0.0.smithi089.dcjmqc smithi089 *:2049 running (7m) 10s ago 7m 286M - 4.0 e1d6a67b021e 98550f94e773 2024-09-21T04:17:02.253 INFO:teuthology.orchestra.run.smithi089.stdout:nfs.foo.1.0.smithi195.vhoayp smithi195 *:2049 running (7m) 100s ago 7m 88.0M - 4.0 e1d6a67b021e c60a4040130a 2024-09-21T04:17:02.253 INFO:teuthology.orchestra.run.smithi089.stdout:node-exporter.smithi089 smithi089 *:9100 running (4m) 10s ago 13m 9991k - 1.7.0 72c9c2088986 7ef0d1588475 2024-09-21T04:17:02.253 INFO:teuthology.orchestra.run.smithi089.stdout:node-exporter.smithi195 smithi195 *:9100 running (4m) 100s ago 13m 10.7M - 1.7.0 72c9c2088986 61eafd61bc6c 2024-09-21T04:17:02.253 INFO:teuthology.orchestra.run.smithi089.stdout:osd.0 smithi089 running (70s) 10s ago 12m 239M 1935M 19.3.0-5103-gf36283e7 48ed0a2cf89d 3df0adea59fb 2024-09-21T04:17:02.253 INFO:teuthology.orchestra.run.smithi089.stdout:osd.1 smithi089 running (20s) 10s ago 12m 12.2M 1935M 19.3.0-5103-gf36283e7 48ed0a2cf89d da2e49774e7e 2024-09-21T04:17:02.253 INFO:teuthology.orchestra.run.smithi089.stdout:osd.2 smithi089 running (11m) 10s ago 11m 485M 1935M 17.2.0 e1d6a67b021e cfa000948859 2024-09-21T04:17:02.253 INFO:teuthology.orchestra.run.smithi089.stdout:osd.3 smithi089 running (11m) 10s ago 11m 579M 1935M 17.2.0 e1d6a67b021e 31c610ed5815 2024-09-21T04:17:02.253 INFO:teuthology.orchestra.run.smithi089.stdout:osd.4 smithi195 running (11m) 100s ago 10m 524M 2704M 17.2.0 e1d6a67b021e f00e18a1f220 2024-09-21T04:17:02.253 INFO:teuthology.orchestra.run.smithi089.stdout:osd.5 smithi195 running (10m) 100s ago 10m 318M 2704M 17.2.0 e1d6a67b021e cecfd6ff1eee 2024-09-21T04:17:02.253 INFO:teuthology.orchestra.run.smithi089.stdout:osd.6 smithi195 running (10m) 100s ago 10m 510M 2704M 17.2.0 e1d6a67b021e 0d7db362cc23 2024-09-21T04:17:02.253 INFO:teuthology.orchestra.run.smithi089.stdout:osd.7 smithi195 running (9m) 100s ago 9m 387M 2704M 17.2.0 e1d6a67b021e 0109b3fea188 2024-09-21T04:17:02.254 INFO:teuthology.orchestra.run.smithi089.stdout:prometheus.smithi089 smithi089 *:9095 running (3m) 10s ago 13m 50.9M - 2.51.0 1d3b7f56885b 8c9f309a6459 2024-09-21T04:17:02.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:17:01 smithi195 ceph-mon[49435]: from='client.34258 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T04:17:02.367 INFO:teuthology.orchestra.run.smithi089.stdout: 5 190015 9.90 MB/sec execute 341 sec latency 63.806 ms 2024-09-21T04:17:02.930 INFO:teuthology.orchestra.run.smithi089.stdout:{ 2024-09-21T04:17:02.930 INFO:teuthology.orchestra.run.smithi089.stdout: "mon": { 2024-09-21T04:17:02.930 INFO:teuthology.orchestra.run.smithi089.stdout: "ceph version 19.3.0-5103-gf36283e7 (f36283e7c91cac1c01cb9a7c4ff21bd697dbd4f2) squid (dev)": 2 2024-09-21T04:17:02.931 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2024-09-21T04:17:02.931 INFO:teuthology.orchestra.run.smithi089.stdout: "mgr": { 2024-09-21T04:17:02.931 INFO:teuthology.orchestra.run.smithi089.stdout: "ceph version 19.3.0-5103-gf36283e7 (f36283e7c91cac1c01cb9a7c4ff21bd697dbd4f2) squid (dev)": 2 2024-09-21T04:17:02.931 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2024-09-21T04:17:02.931 INFO:teuthology.orchestra.run.smithi089.stdout: "osd": { 2024-09-21T04:17:02.931 INFO:teuthology.orchestra.run.smithi089.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 6, 2024-09-21T04:17:02.931 INFO:teuthology.orchestra.run.smithi089.stdout: "ceph version 19.3.0-5103-gf36283e7 (f36283e7c91cac1c01cb9a7c4ff21bd697dbd4f2) squid (dev)": 2 2024-09-21T04:17:02.931 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2024-09-21T04:17:02.931 INFO:teuthology.orchestra.run.smithi089.stdout: "mds": { 2024-09-21T04:17:02.931 INFO:teuthology.orchestra.run.smithi089.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-21T04:17:02.931 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2024-09-21T04:17:02.931 INFO:teuthology.orchestra.run.smithi089.stdout: "overall": { 2024-09-21T04:17:02.931 INFO:teuthology.orchestra.run.smithi089.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8, 2024-09-21T04:17:02.931 INFO:teuthology.orchestra.run.smithi089.stdout: "ceph version 19.3.0-5103-gf36283e7 (f36283e7c91cac1c01cb9a7c4ff21bd697dbd4f2) squid (dev)": 6 2024-09-21T04:17:02.932 INFO:teuthology.orchestra.run.smithi089.stdout: } 2024-09-21T04:17:02.932 INFO:teuthology.orchestra.run.smithi089.stdout:} 2024-09-21T04:17:02.981 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:17:02 smithi089 ceph-mon[64076]: from='client.34260 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T04:17:02.982 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:17:02 smithi089 ceph-mon[64076]: pgmap v112: 97 pgs: 4 peering, 1 active+recovering+undersized+degraded+remapped, 92 active+clean; 324 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 7.1 KiB/s rd, 3.4 MiB/s wr, 350 op/s; 9/1593 objects degraded (0.565%); 1.0 MiB/s, 55 keys/s, 46 objects/s recovering 2024-09-21T04:17:02.982 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:17:02 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:17:02.982 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:17:02 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T04:17:03.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:17:02 smithi195 ceph-mon[49435]: from='client.34260 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T04:17:03.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:17:02 smithi195 ceph-mon[49435]: pgmap v112: 97 pgs: 4 peering, 1 active+recovering+undersized+degraded+remapped, 92 active+clean; 324 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 7.1 KiB/s rd, 3.4 MiB/s wr, 350 op/s; 9/1593 objects degraded (0.565%); 1.0 MiB/s, 55 keys/s, 46 objects/s recovering 2024-09-21T04:17:03.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:17:02 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:17:03.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:17:02 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T04:17:03.367 INFO:teuthology.orchestra.run.smithi089.stdout: 5 190289 9.89 MB/sec execute 342 sec latency 80.075 ms 2024-09-21T04:17:03.490 INFO:teuthology.orchestra.run.smithi089.stdout:{ 2024-09-21T04:17:03.490 INFO:teuthology.orchestra.run.smithi089.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:2195e5f178be0a9390728e2a14088236d8fe6f82635f5c84418fe37a04457e00", 2024-09-21T04:17:03.490 INFO:teuthology.orchestra.run.smithi089.stdout: "in_progress": true, 2024-09-21T04:17:03.490 INFO:teuthology.orchestra.run.smithi089.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-21T04:17:03.490 INFO:teuthology.orchestra.run.smithi089.stdout: "services_complete": [ 2024-09-21T04:17:03.490 INFO:teuthology.orchestra.run.smithi089.stdout: "mgr", 2024-09-21T04:17:03.490 INFO:teuthology.orchestra.run.smithi089.stdout: "crash", 2024-09-21T04:17:03.490 INFO:teuthology.orchestra.run.smithi089.stdout: "mon" 2024-09-21T04:17:03.490 INFO:teuthology.orchestra.run.smithi089.stdout: ], 2024-09-21T04:17:03.490 INFO:teuthology.orchestra.run.smithi089.stdout: "progress": "8/23 daemons upgraded", 2024-09-21T04:17:03.491 INFO:teuthology.orchestra.run.smithi089.stdout: "message": "Currently upgrading osd daemons", 2024-09-21T04:17:03.491 INFO:teuthology.orchestra.run.smithi089.stdout: "is_paused": false 2024-09-21T04:17:03.491 INFO:teuthology.orchestra.run.smithi089.stdout:} 2024-09-21T04:17:03.935 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:17:03 smithi089 ceph-mon[64076]: from='client.34264 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T04:17:03.935 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:17:03 smithi089 ceph-mon[64076]: from='client.? 172.21.15.89:0/2675976200' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T04:17:04.171 INFO:teuthology.orchestra.run.smithi089.stdout:HEALTH_WARN Degraded data redundancy: 9/1593 objects degraded (0.565%), 1 pg degraded 2024-09-21T04:17:04.171 INFO:teuthology.orchestra.run.smithi089.stdout:[WRN] PG_DEGRADED: Degraded data redundancy: 9/1593 objects degraded (0.565%), 1 pg degraded 2024-09-21T04:17:04.171 INFO:teuthology.orchestra.run.smithi089.stdout: pg 3.16 is active+recovering+undersized+degraded+remapped, acting [5,7] 2024-09-21T04:17:04.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:17:03 smithi195 ceph-mon[49435]: from='client.34264 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T04:17:04.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:17:03 smithi195 ceph-mon[49435]: from='client.? 172.21.15.89:0/2675976200' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T04:17:04.367 INFO:teuthology.orchestra.run.smithi089.stdout: 5 190626 9.88 MB/sec execute 343 sec latency 66.690 ms 2024-09-21T04:17:04.952 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:17:04 smithi089 ceph-mon[64076]: from='client.34272 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T04:17:04.952 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:17:04 smithi089 ceph-mon[64076]: pgmap v113: 97 pgs: 97 active+clean; 318 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 5.6 MiB/s wr, 616 op/s; 860 KiB/s, 46 keys/s, 44 objects/s recovering 2024-09-21T04:17:04.952 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:17:04 smithi089 ceph-mon[64076]: from='client.? 172.21.15.89:0/1101582774' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-21T04:17:04.952 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:17:04 smithi089 ceph-mon[64076]: Health check update: Degraded data redundancy: 9/1593 objects degraded (0.565%), 1 pg degraded (PG_DEGRADED) 2024-09-21T04:17:05.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:17:04 smithi195 ceph-mon[49435]: from='client.34272 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T04:17:05.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:17:04 smithi195 ceph-mon[49435]: pgmap v113: 97 pgs: 97 active+clean; 318 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 5.6 MiB/s wr, 616 op/s; 860 KiB/s, 46 keys/s, 44 objects/s recovering 2024-09-21T04:17:05.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:17:04 smithi195 ceph-mon[49435]: from='client.? 172.21.15.89:0/1101582774' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-21T04:17:05.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:17:04 smithi195 ceph-mon[49435]: Health check update: Degraded data redundancy: 9/1593 objects degraded (0.565%), 1 pg degraded (PG_DEGRADED) 2024-09-21T04:17:05.367 INFO:teuthology.orchestra.run.smithi089.stdout: 5 191063 9.89 MB/sec execute 344 sec latency 43.006 ms 2024-09-21T04:17:05.957 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:17:05 smithi089 ceph-mon[64076]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 9/1593 objects degraded (0.565%), 1 pg degraded) 2024-09-21T04:17:05.957 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:17:05 smithi089 ceph-mon[64076]: Cluster is now healthy 2024-09-21T04:17:06.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:17:05 smithi195 ceph-mon[49435]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 9/1593 objects degraded (0.565%), 1 pg degraded) 2024-09-21T04:17:06.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:17:05 smithi195 ceph-mon[49435]: Cluster is now healthy 2024-09-21T04:17:06.367 INFO:teuthology.orchestra.run.smithi089.stdout: 5 191537 9.91 MB/sec execute 345 sec latency 51.983 ms 2024-09-21T04:17:06.961 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:17:06 smithi089 ceph-mon[64076]: pgmap v114: 97 pgs: 97 active+clean; 318 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 8.6 KiB/s rd, 3.4 MiB/s wr, 372 op/s; 653 KiB/s, 35 keys/s, 23 objects/s recovering 2024-09-21T04:17:07.260 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:17:06 smithi195 ceph-mon[49435]: pgmap v114: 97 pgs: 97 active+clean; 318 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 8.6 KiB/s rd, 3.4 MiB/s wr, 372 op/s; 653 KiB/s, 35 keys/s, 23 objects/s recovering 2024-09-21T04:17:07.367 INFO:teuthology.orchestra.run.smithi089.stdout: 5 191915 9.93 MB/sec execute 346 sec latency 35.607 ms 2024-09-21T04:17:08.367 INFO:teuthology.orchestra.run.smithi089.stdout: 5 192209 9.91 MB/sec execute 347 sec latency 53.806 ms 2024-09-21T04:17:09.258 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:17:09 smithi089 ceph-mon[64076]: pgmap v115: 97 pgs: 97 active+clean; 317 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 3.6 KiB/s rd, 3.0 MiB/s wr, 266 op/s; 125 KiB/s, 7 objects/s recovering 2024-09-21T04:17:09.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:17:09 smithi195 ceph-mon[49435]: pgmap v115: 97 pgs: 97 active+clean; 317 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 3.6 KiB/s rd, 3.0 MiB/s wr, 266 op/s; 125 KiB/s, 7 objects/s recovering 2024-09-21T04:17:09.367 INFO:teuthology.orchestra.run.smithi089.stdout: 5 192434 9.90 MB/sec execute 348 sec latency 50.236 ms 2024-09-21T04:17:10.367 INFO:teuthology.orchestra.run.smithi089.stdout: 5 192666 9.88 MB/sec execute 349 sec latency 52.998 ms 2024-09-21T04:17:11.042 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:17:11 smithi089 ceph-mon[64076]: pgmap v116: 97 pgs: 97 active+clean; 328 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.8 MiB/s wr, 461 op/s; 0 B/s, 3 objects/s recovering 2024-09-21T04:17:11.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:17:11 smithi195 ceph-mon[49435]: pgmap v116: 97 pgs: 97 active+clean; 328 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.8 MiB/s wr, 461 op/s; 0 B/s, 3 objects/s recovering 2024-09-21T04:17:11.367 INFO:teuthology.orchestra.run.smithi089.stdout: 5 192963 9.87 MB/sec execute 350 sec latency 53.922 ms 2024-09-21T04:17:12.367 INFO:teuthology.orchestra.run.smithi089.stdout: 5 193233 9.86 MB/sec execute 351 sec latency 58.705 ms 2024-09-21T04:17:13.079 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:17:13 smithi089 ceph-mon[64076]: pgmap v117: 97 pgs: 97 active+clean; 328 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 5.2 KiB/s rd, 4.0 MiB/s wr, 387 op/s; 0 B/s, 2 objects/s recovering 2024-09-21T04:17:13.079 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:17:13 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:17:13.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:17:13 smithi195 ceph-mon[49435]: pgmap v117: 97 pgs: 97 active+clean; 328 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 5.2 KiB/s rd, 4.0 MiB/s wr, 387 op/s; 0 B/s, 2 objects/s recovering 2024-09-21T04:17:13.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:17:13 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:17:13.367 INFO:teuthology.orchestra.run.smithi089.stdout: 5 193589 9.87 MB/sec execute 352 sec latency 40.159 ms 2024-09-21T04:17:14.109 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:17:14 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16} : dispatch 2024-09-21T04:17:14.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:17:14 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16} : dispatch 2024-09-21T04:17:14.367 INFO:teuthology.orchestra.run.smithi089.stdout: 5 193926 9.86 MB/sec execute 353 sec latency 53.762 ms 2024-09-21T04:17:15.129 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:17:15 smithi089 ceph-mon[64076]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-21T04:17:15.129 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:17:15 smithi089 ceph-mon[64076]: Upgrade: osd.2 is safe to restart 2024-09-21T04:17:15.129 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:17:15 smithi089 ceph-mon[64076]: pgmap v118: 97 pgs: 97 active+clean; 323 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 8.9 KiB/s rd, 5.6 MiB/s wr, 563 op/s; 0 B/s, 2 objects/s recovering 2024-09-21T04:17:15.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:17:15 smithi195 ceph-mon[49435]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-21T04:17:15.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:17:15 smithi195 ceph-mon[49435]: Upgrade: osd.2 is safe to restart 2024-09-21T04:17:15.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:17:15 smithi195 ceph-mon[49435]: pgmap v118: 97 pgs: 97 active+clean; 323 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 8.9 KiB/s rd, 5.6 MiB/s wr, 563 op/s; 0 B/s, 2 objects/s recovering 2024-09-21T04:17:15.367 INFO:teuthology.orchestra.run.smithi089.stdout: 5 194302 9.85 MB/sec execute 354 sec latency 48.996 ms 2024-09-21T04:17:16.367 INFO:teuthology.orchestra.run.smithi089.stdout: 5 194628 9.85 MB/sec execute 355 sec latency 39.284 ms 2024-09-21T04:17:17.166 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:17:16 smithi089 ceph-mon[64076]: pgmap v119: 97 pgs: 97 active+clean; 323 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.2 MiB/s wr, 399 op/s 2024-09-21T04:17:17.166 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:17:16 smithi089 ceph-mon[64076]: Upgrade: Updating osd.2 2024-09-21T04:17:17.166 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:17:16 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:17:17.166 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:17:16 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "auth get", "entity": "osd.2"} : dispatch 2024-09-21T04:17:17.166 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:17:16 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T04:17:17.166 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:17:16 smithi089 ceph-mon[64076]: Deploying daemon osd.2 on smithi089 2024-09-21T04:17:17.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:17:16 smithi195 ceph-mon[49435]: pgmap v119: 97 pgs: 97 active+clean; 323 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.2 MiB/s wr, 399 op/s 2024-09-21T04:17:17.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:17:16 smithi195 ceph-mon[49435]: Upgrade: Updating osd.2 2024-09-21T04:17:17.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:17:16 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:17:17.360 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:17:16 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "auth get", "entity": "osd.2"} : dispatch 2024-09-21T04:17:17.360 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:17:16 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T04:17:17.360 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:17:16 smithi195 ceph-mon[49435]: Deploying daemon osd.2 on smithi089 2024-09-21T04:17:17.368 INFO:teuthology.orchestra.run.smithi089.stdout: 5 194975 9.86 MB/sec execute 356 sec latency 120.682 ms 2024-09-21T04:17:18.367 INFO:teuthology.orchestra.run.smithi089.stdout: 5 195328 9.87 MB/sec execute 357 sec latency 63.032 ms 2024-09-21T04:17:18.456 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:17:18 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:17:18.456 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:17:18 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T04:17:18.456 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:17:18 smithi089 ceph-mon[64076]: pgmap v120: 97 pgs: 97 active+clean; 318 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 5.0 MiB/s wr, 464 op/s 2024-09-21T04:17:18.860 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:17:18 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:17:18.860 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:17:18 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T04:17:18.860 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:17:18 smithi195 ceph-mon[49435]: pgmap v120: 97 pgs: 97 active+clean; 318 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 5.0 MiB/s wr, 464 op/s 2024-09-21T04:17:19.368 INFO:teuthology.orchestra.run.smithi089.stdout: 5 195679 9.88 MB/sec execute 358 sec latency 72.963 ms 2024-09-21T04:17:20.368 INFO:teuthology.orchestra.run.smithi089.stdout: 5 195986 9.86 MB/sec execute 359 sec latency 67.923 ms 2024-09-21T04:17:21.255 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:17:21 smithi089 ceph-mon[64076]: pgmap v121: 97 pgs: 97 active+clean; 328 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 8.7 KiB/s rd, 5.8 MiB/s wr, 597 op/s 2024-09-21T04:17:21.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:17:21 smithi195 ceph-mon[49435]: pgmap v121: 97 pgs: 97 active+clean; 328 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 8.7 KiB/s rd, 5.8 MiB/s wr, 597 op/s 2024-09-21T04:17:21.367 INFO:teuthology.orchestra.run.smithi089.stdout: 5 196231 9.84 MB/sec execute 360 sec latency 57.852 ms 2024-09-21T04:17:21.507 INFO:journalctl@ceph.osd.2.smithi089.stdout:Sep 21 04:17:21 smithi089 systemd[1]: Stopping Ceph osd.2 for 1b9b79fc-77ce-11ef-baf5-efdc52797490... 2024-09-21T04:17:21.765 INFO:journalctl@ceph.osd.2.smithi089.stdout:Sep 21 04:17:21 smithi089 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-2[40846]: 2024-09-21T04:17:21.657+0000 7f557cbc5700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.2 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-21T04:17:21.765 INFO:journalctl@ceph.osd.2.smithi089.stdout:Sep 21 04:17:21 smithi089 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-2[40846]: 2024-09-21T04:17:21.657+0000 7f557cbc5700 -1 osd.2 93 *** Got signal Terminated *** 2024-09-21T04:17:21.765 INFO:journalctl@ceph.osd.2.smithi089.stdout:Sep 21 04:17:21 smithi089 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-2[40846]: 2024-09-21T04:17:21.657+0000 7f557cbc5700 -1 osd.2 93 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-21T04:17:22.035 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:17:22 smithi089 ceph-mon[64076]: osd.2 marked itself down and dead 2024-09-21T04:17:22.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:17:22 smithi195 ceph-mon[49435]: osd.2 marked itself down and dead 2024-09-21T04:17:22.367 INFO:teuthology.orchestra.run.smithi089.stdout: 5 196449 9.84 MB/sec execute 361 sec latency 283.165 ms 2024-09-21T04:17:22.985 INFO:journalctl@ceph.osd.2.smithi089.stdout:Sep 21 04:17:22 smithi089 podman[73369]: 2024-09-21 04:17:22.674183771 +0000 UTC m=+1.140376717 container died cfa000948859585afcbf55a5cc140bd125abd1be5ff2ab5dad4195c609a505b5 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-2, io.openshift.expose-services=, maintainer=Guillaume Abrioux , distribution-scope=public, release=754, CEPH_POINT_RELEASE=-17.2.0, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_CLEAN=True, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.openshift.tags=base centos centos-stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.buildah.version=1.19.8, name=centos-stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.k8s.display-name=CentOS Stream 8, architecture=x86_64, RELEASE=HEAD, ceph=True, com.redhat.component=centos-stream-container, GIT_BRANCH=HEAD, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-type=git, build-date=2022-05-03T08:36:31.336870, GIT_REPO=https://github.com/ceph/ceph-container.git, version=8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vendor=Red Hat, Inc.) 2024-09-21T04:17:23.239 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:17:23 smithi089 ceph-mon[64076]: pgmap v122: 97 pgs: 97 active+clean; 328 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.0 MiB/s wr, 405 op/s 2024-09-21T04:17:23.239 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:17:23 smithi089 ceph-mon[64076]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-21T04:17:23.239 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:17:23 smithi089 ceph-mon[64076]: osdmap e94: 8 total, 7 up, 8 in 2024-09-21T04:17:23.239 INFO:journalctl@ceph.osd.2.smithi089.stdout:Sep 21 04:17:23 smithi089 podman[73369]: 2024-09-21 04:17:23.209828463 +0000 UTC m=+1.676021412 container cleanup cfa000948859585afcbf55a5cc140bd125abd1be5ff2ab5dad4195c609a505b5 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-2, GIT_BRANCH=HEAD, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, version=8, CEPH_POINT_RELEASE=-17.2.0, RELEASE=HEAD, com.redhat.component=centos-stream-container, build-date=2022-05-03T08:36:31.336870, io.buildah.version=1.19.8, io.k8s.display-name=CentOS Stream 8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_REPO=https://github.com/ceph/ceph-container.git, distribution-scope=public, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, release=754, vcs-type=git, io.openshift.expose-services=, GIT_CLEAN=True, ceph=True, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.tags=base centos centos-stream, vendor=Red Hat, Inc., architecture=x86_64, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, maintainer=Guillaume Abrioux , name=centos-stream) 2024-09-21T04:17:23.239 INFO:journalctl@ceph.osd.2.smithi089.stdout:Sep 21 04:17:23 smithi089 bash[73369]: ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-2 2024-09-21T04:17:23.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:17:23 smithi195 ceph-mon[49435]: pgmap v122: 97 pgs: 97 active+clean; 328 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.0 MiB/s wr, 405 op/s 2024-09-21T04:17:23.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:17:23 smithi195 ceph-mon[49435]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-21T04:17:23.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:17:23 smithi195 ceph-mon[49435]: osdmap e94: 8 total, 7 up, 8 in 2024-09-21T04:17:23.367 INFO:teuthology.orchestra.run.smithi089.stdout: 5 196511 9.81 MB/sec execute 362 sec latency 974.716 ms 2024-09-21T04:17:23.759 INFO:journalctl@ceph.osd.2.smithi089.stdout:Sep 21 04:17:23 smithi089 podman[73386]: 2024-09-21 04:17:23.57832597 +0000 UTC m=+0.897525117 container remove cfa000948859585afcbf55a5cc140bd125abd1be5ff2ab5dad4195c609a505b5 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-2, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=-17.2.0, vendor=Red Hat, Inc., maintainer=Guillaume Abrioux , vcs-type=git, release=754, distribution-scope=public, io.openshift.tags=base centos centos-stream, GIT_REPO=https://github.com/ceph/ceph-container.git, version=8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.openshift.expose-services=, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, ceph=True, RELEASE=HEAD, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.component=centos-stream-container, name=centos-stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, build-date=2022-05-03T08:36:31.336870, GIT_CLEAN=True, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.buildah.version=1.19.8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.k8s.display-name=CentOS Stream 8, architecture=x86_64) 2024-09-21T04:17:24.274 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:17:24 smithi089 ceph-mon[64076]: osdmap e95: 8 total, 7 up, 8 in 2024-09-21T04:17:24.274 INFO:journalctl@ceph.osd.2.smithi089.stdout:Sep 21 04:17:24 smithi089 podman[73473]: 2024-09-21 04:17:23.974261761 +0000 UTC m=+0.018677579 image pull 48ed0a2cf89d6d0371a2c6185a5869a747265f6abd5ac2a74f00cc32d5628721 quay.ceph.io/ceph-ci/ceph@sha256:2195e5f178be0a9390728e2a14088236d8fe6f82635f5c84418fe37a04457e00 2024-09-21T04:17:24.274 INFO:journalctl@ceph.osd.2.smithi089.stdout:Sep 21 04:17:24 smithi089 podman[73473]: 2024-09-21 04:17:24.14506273 +0000 UTC m=+0.189478549 container create da8c5fad3a9b8fe787d1e15181117310f9a6d1928bc16d67c92bc9e6f7571c94 (image=quay.ceph.io/ceph-ci/ceph@sha256:2195e5f178be0a9390728e2a14088236d8fe6f82635f5c84418fe37a04457e00, name=ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-2-deactivate, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-guits-main-2024-09-20-2310-f36283e, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_CLEAN=True, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23) 2024-09-21T04:17:24.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:17:24 smithi195 ceph-mon[49435]: osdmap e95: 8 total, 7 up, 8 in 2024-09-21T04:17:24.368 INFO:teuthology.orchestra.run.smithi089.stdout: 5 196764 9.80 MB/sec execute 363 sec latency 76.975 ms 2024-09-21T04:17:24.781 INFO:journalctl@ceph.osd.2.smithi089.stdout:Sep 21 04:17:24 smithi089 podman[73473]: 2024-09-21 04:17:24.591356276 +0000 UTC m=+0.635772095 container init da8c5fad3a9b8fe787d1e15181117310f9a6d1928bc16d67c92bc9e6f7571c94 (image=quay.ceph.io/ceph-ci/ceph@sha256:2195e5f178be0a9390728e2a14088236d8fe6f82635f5c84418fe37a04457e00, name=ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-2-deactivate, RELEASE=wip-guits-main-2024-09-20-2310-f36283e, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2) 2024-09-21T04:17:24.781 INFO:journalctl@ceph.osd.2.smithi089.stdout:Sep 21 04:17:24 smithi089 podman[73473]: 2024-09-21 04:17:24.605094059 +0000 UTC m=+0.649509873 container start da8c5fad3a9b8fe787d1e15181117310f9a6d1928bc16d67c92bc9e6f7571c94 (image=quay.ceph.io/ceph-ci/ceph@sha256:2195e5f178be0a9390728e2a14088236d8fe6f82635f5c84418fe37a04457e00, name=ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-2-deactivate, RELEASE=wip-guits-main-2024-09-20-2310-f36283e, org.label-schema.vendor=CentOS, GIT_CLEAN=True, io.buildah.version=1.37.2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, CEPH_POINT_RELEASE=, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731) 2024-09-21T04:17:25.033 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:17:25 smithi089 ceph-mon[64076]: pgmap v125: 97 pgs: 4 active+undersized, 18 peering, 4 active+undersized+degraded, 71 active+clean; 326 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 5.6 KiB/s rd, 5.6 MiB/s wr, 540 op/s; 40/1587 objects degraded (2.520%) 2024-09-21T04:17:25.033 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:17:25 smithi089 ceph-mon[64076]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-09-21T04:17:25.033 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:17:25 smithi089 ceph-mon[64076]: Health check failed: Degraded data redundancy: 40/1587 objects degraded (2.520%), 4 pgs degraded (PG_DEGRADED) 2024-09-21T04:17:25.033 INFO:journalctl@ceph.osd.2.smithi089.stdout:Sep 21 04:17:24 smithi089 podman[73473]: 2024-09-21 04:17:24.827838957 +0000 UTC m=+0.872254775 container attach da8c5fad3a9b8fe787d1e15181117310f9a6d1928bc16d67c92bc9e6f7571c94 (image=quay.ceph.io/ceph-ci/ceph@sha256:2195e5f178be0a9390728e2a14088236d8fe6f82635f5c84418fe37a04457e00, name=ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-2-deactivate, RELEASE=wip-guits-main-2024-09-20-2310-f36283e, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, ceph=True, GIT_CLEAN=True, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731) 2024-09-21T04:17:25.033 INFO:journalctl@ceph.osd.2.smithi089.stdout:Sep 21 04:17:24 smithi089 podman[73473]: 2024-09-21 04:17:24.968571085 +0000 UTC m=+1.012986904 container died da8c5fad3a9b8fe787d1e15181117310f9a6d1928bc16d67c92bc9e6f7571c94 (image=quay.ceph.io/ceph-ci/ceph@sha256:2195e5f178be0a9390728e2a14088236d8fe6f82635f5c84418fe37a04457e00, name=ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-2-deactivate, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-guits-main-2024-09-20-2310-f36283e, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, GIT_CLEAN=True, org.label-schema.license=GPLv2, ceph=True, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23) 2024-09-21T04:17:25.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:17:25 smithi195 ceph-mon[49435]: pgmap v125: 97 pgs: 4 active+undersized, 18 peering, 4 active+undersized+degraded, 71 active+clean; 326 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 5.6 KiB/s rd, 5.6 MiB/s wr, 540 op/s; 40/1587 objects degraded (2.520%) 2024-09-21T04:17:25.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:17:25 smithi195 ceph-mon[49435]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-09-21T04:17:25.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:17:25 smithi195 ceph-mon[49435]: Health check failed: Degraded data redundancy: 40/1587 objects degraded (2.520%), 4 pgs degraded (PG_DEGRADED) 2024-09-21T04:17:25.368 INFO:teuthology.orchestra.run.smithi089.stdout: 5 197187 9.81 MB/sec execute 364 sec latency 56.896 ms 2024-09-21T04:17:26.368 INFO:teuthology.orchestra.run.smithi089.stdout: 5 197510 9.80 MB/sec execute 365 sec latency 45.047 ms 2024-09-21T04:17:26.578 INFO:journalctl@ceph.osd.2.smithi089.stdout:Sep 21 04:17:26 smithi089 podman[73495]: 2024-09-21 04:17:26.488059724 +0000 UTC m=+1.512194328 container remove da8c5fad3a9b8fe787d1e15181117310f9a6d1928bc16d67c92bc9e6f7571c94 (image=quay.ceph.io/ceph-ci/ceph@sha256:2195e5f178be0a9390728e2a14088236d8fe6f82635f5c84418fe37a04457e00, name=ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-2-deactivate, RELEASE=wip-guits-main-2024-09-20-2310-f36283e, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, GIT_CLEAN=True, ceph=True, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS) 2024-09-21T04:17:26.578 INFO:journalctl@ceph.osd.2.smithi089.stdout:Sep 21 04:17:26 smithi089 systemd[1]: ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490@osd.2.service: Deactivated successfully. 2024-09-21T04:17:26.578 INFO:journalctl@ceph.osd.2.smithi089.stdout:Sep 21 04:17:26 smithi089 systemd[1]: Stopped Ceph osd.2 for 1b9b79fc-77ce-11ef-baf5-efdc52797490. 2024-09-21T04:17:26.578 INFO:journalctl@ceph.osd.2.smithi089.stdout:Sep 21 04:17:26 smithi089 systemd[1]: ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490@osd.2.service: Consumed 2min 39.586s CPU time. 2024-09-21T04:17:27.092 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:17:27 smithi089 ceph-mon[64076]: pgmap v126: 97 pgs: 4 active+undersized, 18 peering, 4 active+undersized+degraded, 71 active+clean; 326 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 5.6 KiB/s rd, 4.5 MiB/s wr, 443 op/s; 40/1587 objects degraded (2.520%) 2024-09-21T04:17:27.092 INFO:journalctl@ceph.osd.2.smithi089.stdout:Sep 21 04:17:26 smithi089 systemd[1]: Starting Ceph osd.2 for 1b9b79fc-77ce-11ef-baf5-efdc52797490... 2024-09-21T04:17:27.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:17:27 smithi195 ceph-mon[49435]: pgmap v126: 97 pgs: 4 active+undersized, 18 peering, 4 active+undersized+degraded, 71 active+clean; 326 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 5.6 KiB/s rd, 4.5 MiB/s wr, 443 op/s; 40/1587 objects degraded (2.520%) 2024-09-21T04:17:27.368 INFO:journalctl@ceph.osd.2.smithi089.stdout:Sep 21 04:17:27 smithi089 podman[73593]: 2024-09-21 04:17:27.047488658 +0000 UTC m=+0.021441742 image pull 48ed0a2cf89d6d0371a2c6185a5869a747265f6abd5ac2a74f00cc32d5628721 quay.ceph.io/ceph-ci/ceph@sha256:2195e5f178be0a9390728e2a14088236d8fe6f82635f5c84418fe37a04457e00 2024-09-21T04:17:27.368 INFO:journalctl@ceph.osd.2.smithi089.stdout:Sep 21 04:17:27 smithi089 podman[73593]: 2024-09-21 04:17:27.22816128 +0000 UTC m=+0.202114353 container create 2d5671a054d4c3fc7d3460011968dbaf54e4e89458a407a5e35478e0c46a668d (image=quay.ceph.io/ceph-ci/ceph@sha256:2195e5f178be0a9390728e2a14088236d8fe6f82635f5c84418fe37a04457e00, name=ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-2-activate, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, RELEASE=wip-guits-main-2024-09-20-2310-f36283e, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.license=GPLv2, ceph=True, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=) 2024-09-21T04:17:27.368 INFO:teuthology.orchestra.run.smithi089.stdout: 5 197784 9.79 MB/sec execute 366 sec latency 116.282 ms 2024-09-21T04:17:27.643 INFO:journalctl@ceph.osd.2.smithi089.stdout:Sep 21 04:17:27 smithi089 podman[73593]: 2024-09-21 04:17:27.56022014 +0000 UTC m=+0.534173241 container init 2d5671a054d4c3fc7d3460011968dbaf54e4e89458a407a5e35478e0c46a668d (image=quay.ceph.io/ceph-ci/ceph@sha256:2195e5f178be0a9390728e2a14088236d8fe6f82635f5c84418fe37a04457e00, name=ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-2-activate, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.build-date=20240731, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, ceph=True, io.buildah.version=1.37.2, RELEASE=wip-guits-main-2024-09-20-2310-f36283e) 2024-09-21T04:17:27.643 INFO:journalctl@ceph.osd.2.smithi089.stdout:Sep 21 04:17:27 smithi089 podman[73593]: 2024-09-21 04:17:27.567638304 +0000 UTC m=+0.541591386 container start 2d5671a054d4c3fc7d3460011968dbaf54e4e89458a407a5e35478e0c46a668d (image=quay.ceph.io/ceph-ci/ceph@sha256:2195e5f178be0a9390728e2a14088236d8fe6f82635f5c84418fe37a04457e00, name=ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-2-activate, org.label-schema.schema-version=1.0, RELEASE=wip-guits-main-2024-09-20-2310-f36283e, GIT_BRANCH=HEAD, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, ceph=True, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=) 2024-09-21T04:17:27.896 INFO:journalctl@ceph.osd.2.smithi089.stdout:Sep 21 04:17:27 smithi089 podman[73593]: 2024-09-21 04:17:27.671172317 +0000 UTC m=+0.645125391 container attach 2d5671a054d4c3fc7d3460011968dbaf54e4e89458a407a5e35478e0c46a668d (image=quay.ceph.io/ceph-ci/ceph@sha256:2195e5f178be0a9390728e2a14088236d8fe6f82635f5c84418fe37a04457e00, name=ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-2-activate, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-guits-main-2024-09-20-2310-f36283e, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, GIT_CLEAN=True, org.label-schema.license=GPLv2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731) 2024-09-21T04:17:27.896 INFO:journalctl@ceph.osd.2.smithi089.stdout:Sep 21 04:17:27 smithi089 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-2-activate[73608]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T04:17:27.896 INFO:journalctl@ceph.osd.2.smithi089.stdout:Sep 21 04:17:27 smithi089 bash[73593]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T04:17:27.896 INFO:journalctl@ceph.osd.2.smithi089.stdout:Sep 21 04:17:27 smithi089 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-2-activate[73608]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T04:17:27.896 INFO:journalctl@ceph.osd.2.smithi089.stdout:Sep 21 04:17:27 smithi089 bash[73593]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T04:17:28.368 INFO:teuthology.orchestra.run.smithi089.stdout: 5 197977 9.77 MB/sec execute 367 sec latency 72.026 ms 2024-09-21T04:17:29.230 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:17:29 smithi089 ceph-mon[64076]: pgmap v127: 97 pgs: 4 active+undersized, 10 peering, 12 active+undersized+degraded, 71 active+clean; 330 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 1.9 KiB/s rd, 3.6 MiB/s wr, 403 op/s; 91/1575 objects degraded (5.778%) 2024-09-21T04:17:29.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:17:29 smithi195 ceph-mon[49435]: pgmap v127: 97 pgs: 4 active+undersized, 10 peering, 12 active+undersized+degraded, 71 active+clean; 330 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 1.9 KiB/s rd, 3.6 MiB/s wr, 403 op/s; 91/1575 objects degraded (5.778%) 2024-09-21T04:17:29.368 INFO:teuthology.orchestra.run.smithi089.stdout: 5 198173 9.76 MB/sec execute 368 sec latency 87.928 ms 2024-09-21T04:17:30.275 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:17:30 smithi089 ceph-mon[64076]: Health check update: Degraded data redundancy: 91/1575 objects degraded (5.778%), 12 pgs degraded (PG_DEGRADED) 2024-09-21T04:17:30.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:17:30 smithi195 ceph-mon[49435]: Health check update: Degraded data redundancy: 91/1575 objects degraded (5.778%), 12 pgs degraded (PG_DEGRADED) 2024-09-21T04:17:30.368 INFO:teuthology.orchestra.run.smithi089.stdout: 5 198415 9.76 MB/sec execute 369 sec latency 92.276 ms 2024-09-21T04:17:31.049 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:17:31 smithi089 ceph-mon[64076]: pgmap v128: 97 pgs: 6 active+undersized, 20 active+undersized+degraded, 71 active+clean; 324 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 6.6 KiB/s rd, 4.4 MiB/s wr, 464 op/s; 142/1560 objects degraded (9.103%) 2024-09-21T04:17:31.049 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:17:31 smithi089 ceph-mon[64076]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-09-21T04:17:31.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:17:31 smithi195 ceph-mon[49435]: pgmap v128: 97 pgs: 6 active+undersized, 20 active+undersized+degraded, 71 active+clean; 324 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 6.6 KiB/s rd, 4.4 MiB/s wr, 464 op/s; 142/1560 objects degraded (9.103%) 2024-09-21T04:17:31.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:17:31 smithi195 ceph-mon[49435]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-09-21T04:17:31.368 INFO:teuthology.orchestra.run.smithi089.stdout: 5 198684 9.77 MB/sec execute 370 sec latency 59.773 ms 2024-09-21T04:17:31.806 INFO:journalctl@ceph.osd.2.smithi089.stdout:Sep 21 04:17:31 smithi089 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-2-activate[73608]: --> Failed to activate via raw: did not find any matching OSD to activate 2024-09-21T04:17:31.806 INFO:journalctl@ceph.osd.2.smithi089.stdout:Sep 21 04:17:31 smithi089 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-2-activate[73608]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T04:17:31.806 INFO:journalctl@ceph.osd.2.smithi089.stdout:Sep 21 04:17:31 smithi089 bash[73593]: --> Failed to activate via raw: did not find any matching OSD to activate 2024-09-21T04:17:31.806 INFO:journalctl@ceph.osd.2.smithi089.stdout:Sep 21 04:17:31 smithi089 bash[73593]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T04:17:31.806 INFO:journalctl@ceph.osd.2.smithi089.stdout:Sep 21 04:17:31 smithi089 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-2-activate[73608]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T04:17:31.806 INFO:journalctl@ceph.osd.2.smithi089.stdout:Sep 21 04:17:31 smithi089 bash[73593]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T04:17:32.064 INFO:journalctl@ceph.osd.2.smithi089.stdout:Sep 21 04:17:31 smithi089 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-2-activate[73608]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-09-21T04:17:32.064 INFO:journalctl@ceph.osd.2.smithi089.stdout:Sep 21 04:17:31 smithi089 bash[73593]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-09-21T04:17:32.064 INFO:journalctl@ceph.osd.2.smithi089.stdout:Sep 21 04:17:31 smithi089 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-2-activate[73608]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_2 --path /var/lib/ceph/osd/ceph-2 --no-mon-config 2024-09-21T04:17:32.064 INFO:journalctl@ceph.osd.2.smithi089.stdout:Sep 21 04:17:31 smithi089 bash[73593]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_2 --path /var/lib/ceph/osd/ceph-2 --no-mon-config 2024-09-21T04:17:32.316 INFO:journalctl@ceph.osd.2.smithi089.stdout:Sep 21 04:17:32 smithi089 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-2-activate[73608]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_2 /var/lib/ceph/osd/ceph-2/block 2024-09-21T04:17:32.316 INFO:journalctl@ceph.osd.2.smithi089.stdout:Sep 21 04:17:32 smithi089 bash[73593]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_2 /var/lib/ceph/osd/ceph-2/block 2024-09-21T04:17:32.316 INFO:journalctl@ceph.osd.2.smithi089.stdout:Sep 21 04:17:32 smithi089 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-2-activate[73608]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-2/block 2024-09-21T04:17:32.316 INFO:journalctl@ceph.osd.2.smithi089.stdout:Sep 21 04:17:32 smithi089 bash[73593]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-2/block 2024-09-21T04:17:32.316 INFO:journalctl@ceph.osd.2.smithi089.stdout:Sep 21 04:17:32 smithi089 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-2-activate[73608]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-09-21T04:17:32.317 INFO:journalctl@ceph.osd.2.smithi089.stdout:Sep 21 04:17:32 smithi089 bash[73593]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-09-21T04:17:32.317 INFO:journalctl@ceph.osd.2.smithi089.stdout:Sep 21 04:17:32 smithi089 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-2-activate[73608]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-09-21T04:17:32.317 INFO:journalctl@ceph.osd.2.smithi089.stdout:Sep 21 04:17:32 smithi089 bash[73593]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-09-21T04:17:32.317 INFO:journalctl@ceph.osd.2.smithi089.stdout:Sep 21 04:17:32 smithi089 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-2-activate[73608]: --> ceph-volume lvm activate successful for osd ID: 2 2024-09-21T04:17:32.317 INFO:journalctl@ceph.osd.2.smithi089.stdout:Sep 21 04:17:32 smithi089 bash[73593]: --> ceph-volume lvm activate successful for osd ID: 2 2024-09-21T04:17:32.317 INFO:journalctl@ceph.osd.2.smithi089.stdout:Sep 21 04:17:32 smithi089 podman[73593]: 2024-09-21 04:17:32.30244593 +0000 UTC m=+5.276399006 container died 2d5671a054d4c3fc7d3460011968dbaf54e4e89458a407a5e35478e0c46a668d (image=quay.ceph.io/ceph-ci/ceph@sha256:2195e5f178be0a9390728e2a14088236d8fe6f82635f5c84418fe37a04457e00, name=ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-2-activate, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.license=GPLv2, RELEASE=wip-guits-main-2024-09-20-2310-f36283e, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, ceph=True, io.buildah.version=1.37.2) 2024-09-21T04:17:32.368 INFO:teuthology.orchestra.run.smithi089.stdout: 5 198978 9.77 MB/sec execute 371 sec latency 54.551 ms 2024-09-21T04:17:33.086 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:17:33 smithi089 ceph-mon[64076]: pgmap v129: 97 pgs: 6 active+undersized, 20 active+undersized+degraded, 71 active+clean; 324 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 5.3 KiB/s rd, 3.5 MiB/s wr, 373 op/s; 142/1560 objects degraded (9.103%) 2024-09-21T04:17:33.086 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:17:33 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:17:33.086 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:17:33 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T04:17:33.347 INFO:journalctl@ceph.osd.2.smithi089.stdout:Sep 21 04:17:33 smithi089 podman[73818]: 2024-09-21 04:17:33.1618315 +0000 UTC m=+0.853357058 container remove 2d5671a054d4c3fc7d3460011968dbaf54e4e89458a407a5e35478e0c46a668d (image=quay.ceph.io/ceph-ci/ceph@sha256:2195e5f178be0a9390728e2a14088236d8fe6f82635f5c84418fe37a04457e00, name=ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-2-activate, RELEASE=wip-guits-main-2024-09-20-2310-f36283e, org.label-schema.vendor=CentOS, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, GIT_CLEAN=True, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-21T04:17:33.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:17:33 smithi195 ceph-mon[49435]: pgmap v129: 97 pgs: 6 active+undersized, 20 active+undersized+degraded, 71 active+clean; 324 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 5.3 KiB/s rd, 3.5 MiB/s wr, 373 op/s; 142/1560 objects degraded (9.103%) 2024-09-21T04:17:33.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:17:33 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:17:33.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:17:33 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T04:17:33.368 INFO:teuthology.orchestra.run.smithi089.stdout: 5 199228 9.76 MB/sec execute 372 sec latency 57.784 ms 2024-09-21T04:17:33.449 INFO:journalctl@ceph.osd.2.smithi089.stdout:Sep 21 04:17:33 smithi089 podman[73891]: 2024-09-21 04:17:33.287698782 +0000 UTC m=+0.017213223 image pull 48ed0a2cf89d6d0371a2c6185a5869a747265f6abd5ac2a74f00cc32d5628721 quay.ceph.io/ceph-ci/ceph@sha256:2195e5f178be0a9390728e2a14088236d8fe6f82635f5c84418fe37a04457e00 2024-09-21T04:17:33.449 INFO:journalctl@ceph.osd.2.smithi089.stdout:Sep 21 04:17:33 smithi089 podman[73891]: 2024-09-21 04:17:33.420628645 +0000 UTC m=+0.150143089 container create b9f6438dd5b9af6ba60900e659361b714e9c8e5631963f8c00cdabc7516099a5 (image=quay.ceph.io/ceph-ci/ceph@sha256:2195e5f178be0a9390728e2a14088236d8fe6f82635f5c84418fe37a04457e00, name=ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-2, RELEASE=wip-guits-main-2024-09-20-2310-f36283e, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , ceph=True) 2024-09-21T04:17:33.849 INFO:journalctl@ceph.osd.2.smithi089.stdout:Sep 21 04:17:33 smithi089 podman[73891]: 2024-09-21 04:17:33.781323745 +0000 UTC m=+0.510838182 container init b9f6438dd5b9af6ba60900e659361b714e9c8e5631963f8c00cdabc7516099a5 (image=quay.ceph.io/ceph-ci/ceph@sha256:2195e5f178be0a9390728e2a14088236d8fe6f82635f5c84418fe37a04457e00, name=ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-2, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, GIT_CLEAN=True, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-guits-main-2024-09-20-2310-f36283e, ceph=True, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD) 2024-09-21T04:17:33.849 INFO:journalctl@ceph.osd.2.smithi089.stdout:Sep 21 04:17:33 smithi089 podman[73891]: 2024-09-21 04:17:33.794745264 +0000 UTC m=+0.524259708 container start b9f6438dd5b9af6ba60900e659361b714e9c8e5631963f8c00cdabc7516099a5 (image=quay.ceph.io/ceph-ci/ceph@sha256:2195e5f178be0a9390728e2a14088236d8fe6f82635f5c84418fe37a04457e00, name=ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-2, GIT_CLEAN=True, org.label-schema.license=GPLv2, RELEASE=wip-guits-main-2024-09-20-2310-f36283e, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD) 2024-09-21T04:17:33.849 INFO:journalctl@ceph.osd.2.smithi089.stdout:Sep 21 04:17:33 smithi089 ceph-osd[73909]: -- 172.21.15.89:0/3977211979 <== mon.0 v2:172.21.15.89:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55c1a2f341e0 con 0x55c1a20cfc00 2024-09-21T04:17:34.101 INFO:journalctl@ceph.osd.2.smithi089.stdout:Sep 21 04:17:33 smithi089 bash[73891]: b9f6438dd5b9af6ba60900e659361b714e9c8e5631963f8c00cdabc7516099a5 2024-09-21T04:17:34.101 INFO:journalctl@ceph.osd.2.smithi089.stdout:Sep 21 04:17:33 smithi089 systemd[1]: Started Ceph osd.2 for 1b9b79fc-77ce-11ef-baf5-efdc52797490. 2024-09-21T04:17:34.369 INFO:teuthology.orchestra.run.smithi089.stdout: 5 199460 9.74 MB/sec execute 373 sec latency 75.192 ms 2024-09-21T04:17:34.630 INFO:teuthology.orchestra.run.smithi089.stdout:true 2024-09-21T04:17:35.133 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:17:34 smithi089 ceph-mon[64076]: pgmap v130: 97 pgs: 6 active+undersized, 20 active+undersized+degraded, 71 active+clean; 329 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 5.6 KiB/s rd, 4.3 MiB/s wr, 432 op/s; 147/1659 objects degraded (8.861%) 2024-09-21T04:17:35.134 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:17:34 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:17:35.134 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:17:34 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:17:35.134 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:17:34 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T04:17:35.134 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:17:34 smithi089 ceph-mon[64076]: Health check update: Degraded data redundancy: 147/1659 objects degraded (8.861%), 20 pgs degraded (PG_DEGRADED) 2024-09-21T04:17:35.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:17:34 smithi195 ceph-mon[49435]: pgmap v130: 97 pgs: 6 active+undersized, 20 active+undersized+degraded, 71 active+clean; 329 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 5.6 KiB/s rd, 4.3 MiB/s wr, 432 op/s; 147/1659 objects degraded (8.861%) 2024-09-21T04:17:35.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:17:34 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:17:35.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:17:34 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:17:35.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:17:34 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T04:17:35.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:17:34 smithi195 ceph-mon[49435]: Health check update: Degraded data redundancy: 147/1659 objects degraded (8.861%), 20 pgs degraded (PG_DEGRADED) 2024-09-21T04:17:35.369 INFO:teuthology.orchestra.run.smithi089.stdout: 5 199642 9.72 MB/sec execute 374 sec latency 66.336 ms 2024-09-21T04:17:35.512 INFO:teuthology.orchestra.run.smithi089.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-21T04:17:35.512 INFO:teuthology.orchestra.run.smithi089.stdout:alertmanager.smithi089 smithi089 *:9093,9094 running (5m) 44s ago 14m 17.2M - 0.27.0 11f11916f8cd 65dfcbcafee1 2024-09-21T04:17:35.512 INFO:teuthology.orchestra.run.smithi089.stdout:crash.smithi089 smithi089 running (2m) 44s ago 14m 6891k - 19.3.0-5103-gf36283e7 48ed0a2cf89d 6c415a5e173e 2024-09-21T04:17:35.512 INFO:teuthology.orchestra.run.smithi089.stdout:crash.smithi195 smithi195 running (2m) 2m ago 14m 6891k - 19.3.0-5103-gf36283e7 48ed0a2cf89d 78ce05a4e9ab 2024-09-21T04:17:35.512 INFO:teuthology.orchestra.run.smithi089.stdout:grafana.smithi089 smithi089 *:3000 running (5m) 44s ago 14m 45.4M - dad864ee21e9 ec4d6738bce7 2024-09-21T04:17:35.512 INFO:teuthology.orchestra.run.smithi089.stdout:mds.foofs.smithi089.osufws smithi089 running (8m) 44s ago 8m 38.3M - 17.2.0 e1d6a67b021e 664d41859bea 2024-09-21T04:17:35.512 INFO:teuthology.orchestra.run.smithi089.stdout:mds.foofs.smithi195.hijxre smithi195 running (8m) 2m ago 8m 16.7M - 17.2.0 e1d6a67b021e 7bc0c6542d12 2024-09-21T04:17:35.512 INFO:teuthology.orchestra.run.smithi089.stdout:mgr.smithi089.srlyxf smithi089 *:8443,9283,8765 running (4m) 44s ago 15m 515M - 19.3.0-5103-gf36283e7 48ed0a2cf89d 3cbeaaa88241 2024-09-21T04:17:35.512 INFO:teuthology.orchestra.run.smithi089.stdout:mgr.smithi195.cgotzm smithi195 *:8443,9283,8765 running (3m) 2m ago 14m 454M - 19.3.0-5103-gf36283e7 48ed0a2cf89d d3734574cf66 2024-09-21T04:17:35.512 INFO:teuthology.orchestra.run.smithi089.stdout:mon.smithi089 smithi089 running (3m) 44s ago 16m 49.9M 2048M 19.3.0-5103-gf36283e7 48ed0a2cf89d 7dd34c8a6f21 2024-09-21T04:17:35.512 INFO:teuthology.orchestra.run.smithi089.stdout:mon.smithi195 smithi195 running (3m) 2m ago 14m 36.7M 2048M 19.3.0-5103-gf36283e7 48ed0a2cf89d 4255937444e7 2024-09-21T04:17:35.512 INFO:teuthology.orchestra.run.smithi089.stdout:nfs.foo.0.0.smithi089.dcjmqc smithi089 *:2049 running (8m) 44s ago 8m 286M - 4.0 e1d6a67b021e 98550f94e773 2024-09-21T04:17:35.512 INFO:teuthology.orchestra.run.smithi089.stdout:nfs.foo.1.0.smithi195.vhoayp smithi195 *:2049 running (8m) 2m ago 8m 88.0M - 4.0 e1d6a67b021e c60a4040130a 2024-09-21T04:17:35.513 INFO:teuthology.orchestra.run.smithi089.stdout:node-exporter.smithi089 smithi089 *:9100 running (5m) 44s ago 14m 9991k - 1.7.0 72c9c2088986 7ef0d1588475 2024-09-21T04:17:35.513 INFO:teuthology.orchestra.run.smithi089.stdout:node-exporter.smithi195 smithi195 *:9100 running (5m) 2m ago 13m 10.7M - 1.7.0 72c9c2088986 61eafd61bc6c 2024-09-21T04:17:35.513 INFO:teuthology.orchestra.run.smithi089.stdout:osd.0 smithi089 running (104s) 44s ago 13m 239M 1935M 19.3.0-5103-gf36283e7 48ed0a2cf89d 3df0adea59fb 2024-09-21T04:17:35.513 INFO:teuthology.orchestra.run.smithi089.stdout:osd.1 smithi089 running (53s) 44s ago 12m 12.2M 1935M 19.3.0-5103-gf36283e7 48ed0a2cf89d da2e49774e7e 2024-09-21T04:17:35.513 INFO:teuthology.orchestra.run.smithi089.stdout:osd.2 smithi089 starting - - - 1935M 2024-09-21T04:17:35.513 INFO:teuthology.orchestra.run.smithi089.stdout:osd.3 smithi089 running (11m) 44s ago 11m 579M 1935M 17.2.0 e1d6a67b021e 31c610ed5815 2024-09-21T04:17:35.513 INFO:teuthology.orchestra.run.smithi089.stdout:osd.4 smithi195 running (11m) 2m ago 11m 524M 2704M 17.2.0 e1d6a67b021e f00e18a1f220 2024-09-21T04:17:35.513 INFO:teuthology.orchestra.run.smithi089.stdout:osd.5 smithi195 running (11m) 2m ago 11m 318M 2704M 17.2.0 e1d6a67b021e cecfd6ff1eee 2024-09-21T04:17:35.513 INFO:teuthology.orchestra.run.smithi089.stdout:osd.6 smithi195 running (10m) 2m ago 10m 510M 2704M 17.2.0 e1d6a67b021e 0d7db362cc23 2024-09-21T04:17:35.513 INFO:teuthology.orchestra.run.smithi089.stdout:osd.7 smithi195 running (10m) 2m ago 10m 387M 2704M 17.2.0 e1d6a67b021e 0109b3fea188 2024-09-21T04:17:35.513 INFO:teuthology.orchestra.run.smithi089.stdout:prometheus.smithi089 smithi089 *:9095 running (4m) 44s ago 13m 50.9M - 2.51.0 1d3b7f56885b 8c9f309a6459 2024-09-21T04:17:35.906 INFO:journalctl@ceph.osd.2.smithi089.stdout:Sep 21 04:17:35 smithi089 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-2[73905]: 2024-09-21T04:17:35.869+0000 7f4f1ee52640 -1 Falling back to public interface 2024-09-21T04:17:36.096 INFO:teuthology.orchestra.run.smithi089.stdout:{ 2024-09-21T04:17:36.096 INFO:teuthology.orchestra.run.smithi089.stdout: "mon": { 2024-09-21T04:17:36.096 INFO:teuthology.orchestra.run.smithi089.stdout: "ceph version 19.3.0-5103-gf36283e7 (f36283e7c91cac1c01cb9a7c4ff21bd697dbd4f2) squid (dev)": 2 2024-09-21T04:17:36.096 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2024-09-21T04:17:36.096 INFO:teuthology.orchestra.run.smithi089.stdout: "mgr": { 2024-09-21T04:17:36.096 INFO:teuthology.orchestra.run.smithi089.stdout: "ceph version 19.3.0-5103-gf36283e7 (f36283e7c91cac1c01cb9a7c4ff21bd697dbd4f2) squid (dev)": 2 2024-09-21T04:17:36.096 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2024-09-21T04:17:36.096 INFO:teuthology.orchestra.run.smithi089.stdout: "osd": { 2024-09-21T04:17:36.096 INFO:teuthology.orchestra.run.smithi089.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 5, 2024-09-21T04:17:36.097 INFO:teuthology.orchestra.run.smithi089.stdout: "ceph version 19.3.0-5103-gf36283e7 (f36283e7c91cac1c01cb9a7c4ff21bd697dbd4f2) squid (dev)": 2 2024-09-21T04:17:36.097 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2024-09-21T04:17:36.097 INFO:teuthology.orchestra.run.smithi089.stdout: "mds": { 2024-09-21T04:17:36.097 INFO:teuthology.orchestra.run.smithi089.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-21T04:17:36.097 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2024-09-21T04:17:36.097 INFO:teuthology.orchestra.run.smithi089.stdout: "overall": { 2024-09-21T04:17:36.097 INFO:teuthology.orchestra.run.smithi089.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 7, 2024-09-21T04:17:36.097 INFO:teuthology.orchestra.run.smithi089.stdout: "ceph version 19.3.0-5103-gf36283e7 (f36283e7c91cac1c01cb9a7c4ff21bd697dbd4f2) squid (dev)": 6 2024-09-21T04:17:36.097 INFO:teuthology.orchestra.run.smithi089.stdout: } 2024-09-21T04:17:36.097 INFO:teuthology.orchestra.run.smithi089.stdout:} 2024-09-21T04:17:36.157 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:17:35 smithi089 ceph-mon[64076]: from='client.34282 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T04:17:36.157 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:17:35 smithi089 ceph-mon[64076]: from='client.34286 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T04:17:36.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:17:35 smithi195 ceph-mon[49435]: from='client.34282 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T04:17:36.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:17:35 smithi195 ceph-mon[49435]: from='client.34286 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T04:17:36.369 INFO:teuthology.orchestra.run.smithi089.stdout: 5 199985 9.72 MB/sec execute 375 sec latency 56.261 ms 2024-09-21T04:17:36.542 INFO:teuthology.orchestra.run.smithi089.stdout:{ 2024-09-21T04:17:36.542 INFO:teuthology.orchestra.run.smithi089.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:2195e5f178be0a9390728e2a14088236d8fe6f82635f5c84418fe37a04457e00", 2024-09-21T04:17:36.542 INFO:teuthology.orchestra.run.smithi089.stdout: "in_progress": true, 2024-09-21T04:17:36.542 INFO:teuthology.orchestra.run.smithi089.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-21T04:17:36.542 INFO:teuthology.orchestra.run.smithi089.stdout: "services_complete": [ 2024-09-21T04:17:36.542 INFO:teuthology.orchestra.run.smithi089.stdout: "mgr", 2024-09-21T04:17:36.542 INFO:teuthology.orchestra.run.smithi089.stdout: "crash", 2024-09-21T04:17:36.542 INFO:teuthology.orchestra.run.smithi089.stdout: "mon" 2024-09-21T04:17:36.542 INFO:teuthology.orchestra.run.smithi089.stdout: ], 2024-09-21T04:17:36.543 INFO:teuthology.orchestra.run.smithi089.stdout: "progress": "8/23 daemons upgraded", 2024-09-21T04:17:36.543 INFO:teuthology.orchestra.run.smithi089.stdout: "message": "Currently upgrading osd daemons", 2024-09-21T04:17:36.543 INFO:teuthology.orchestra.run.smithi089.stdout: "is_paused": false 2024-09-21T04:17:36.543 INFO:teuthology.orchestra.run.smithi089.stdout:} 2024-09-21T04:17:37.045 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:17:36 smithi089 ceph-mon[64076]: from='client.44205 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T04:17:37.045 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:17:37 smithi089 ceph-mon[64076]: pgmap v131: 97 pgs: 6 active+undersized, 20 active+undersized+degraded, 71 active+clean; 329 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 5.1 KiB/s rd, 3.3 MiB/s wr, 326 op/s; 147/1659 objects degraded (8.861%) 2024-09-21T04:17:37.045 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:17:37 smithi089 ceph-mon[64076]: from='client.? 172.21.15.89:0/2510119759' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T04:17:37.130 INFO:teuthology.orchestra.run.smithi089.stdout:HEALTH_WARN 1 osds down; Degraded data redundancy: 147/1659 objects degraded (8.861%), 20 pgs degraded 2024-09-21T04:17:37.130 INFO:teuthology.orchestra.run.smithi089.stdout:[WRN] OSD_DOWN: 1 osds down 2024-09-21T04:17:37.130 INFO:teuthology.orchestra.run.smithi089.stdout: osd.2 (root=default,host=smithi089) is down 2024-09-21T04:17:37.130 INFO:teuthology.orchestra.run.smithi089.stdout:[WRN] PG_DEGRADED: Degraded data redundancy: 147/1659 objects degraded (8.861%), 20 pgs degraded 2024-09-21T04:17:37.130 INFO:teuthology.orchestra.run.smithi089.stdout: pg 2.1 is active+undersized+degraded, acting [3,0] 2024-09-21T04:17:37.130 INFO:teuthology.orchestra.run.smithi089.stdout: pg 2.3 is active+undersized+degraded, acting [5,7] 2024-09-21T04:17:37.130 INFO:teuthology.orchestra.run.smithi089.stdout: pg 2.7 is active+undersized+degraded, acting [6,7] 2024-09-21T04:17:37.130 INFO:teuthology.orchestra.run.smithi089.stdout: pg 2.b is active+undersized+degraded, acting [4,5] 2024-09-21T04:17:37.130 INFO:teuthology.orchestra.run.smithi089.stdout: pg 2.c is active+undersized+degraded, acting [5,0] 2024-09-21T04:17:37.131 INFO:teuthology.orchestra.run.smithi089.stdout: pg 2.e is active+undersized+degraded, acting [3,7] 2024-09-21T04:17:37.131 INFO:teuthology.orchestra.run.smithi089.stdout: pg 2.10 is active+undersized+degraded, acting [1,0] 2024-09-21T04:17:37.131 INFO:teuthology.orchestra.run.smithi089.stdout: pg 2.16 is active+undersized+degraded, acting [5,6] 2024-09-21T04:17:37.131 INFO:teuthology.orchestra.run.smithi089.stdout: pg 2.17 is active+undersized+degraded, acting [6,5] 2024-09-21T04:17:37.131 INFO:teuthology.orchestra.run.smithi089.stdout: pg 2.1c is active+undersized+degraded, acting [7,5] 2024-09-21T04:17:37.131 INFO:teuthology.orchestra.run.smithi089.stdout: pg 3.0 is active+undersized+degraded, acting [1,6] 2024-09-21T04:17:37.131 INFO:teuthology.orchestra.run.smithi089.stdout: pg 3.4 is active+undersized+degraded, acting [1,5] 2024-09-21T04:17:37.131 INFO:teuthology.orchestra.run.smithi089.stdout: pg 3.5 is active+undersized+degraded, acting [5,3] 2024-09-21T04:17:37.131 INFO:teuthology.orchestra.run.smithi089.stdout: pg 3.9 is active+undersized+degraded, acting [4,7] 2024-09-21T04:17:37.131 INFO:teuthology.orchestra.run.smithi089.stdout: pg 3.13 is active+undersized+degraded, acting [7,4] 2024-09-21T04:17:37.131 INFO:teuthology.orchestra.run.smithi089.stdout: pg 3.1a is active+undersized+degraded, acting [4,1] 2024-09-21T04:17:37.131 INFO:teuthology.orchestra.run.smithi089.stdout: pg 3.1e is active+undersized+degraded, acting [3,6] 2024-09-21T04:17:37.131 INFO:teuthology.orchestra.run.smithi089.stdout: pg 3.1f is active+undersized+degraded, acting [0,5] 2024-09-21T04:17:37.131 INFO:teuthology.orchestra.run.smithi089.stdout: pg 4.14 is active+undersized+degraded, acting [3,7] 2024-09-21T04:17:37.131 INFO:teuthology.orchestra.run.smithi089.stdout: pg 4.19 is active+undersized+degraded, acting [3,0] 2024-09-21T04:17:37.260 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:17:37 smithi195 ceph-mon[49435]: from='client.44205 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T04:17:37.260 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:17:37 smithi195 ceph-mon[49435]: pgmap v131: 97 pgs: 6 active+undersized, 20 active+undersized+degraded, 71 active+clean; 329 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 5.1 KiB/s rd, 3.3 MiB/s wr, 326 op/s; 147/1659 objects degraded (8.861%) 2024-09-21T04:17:37.260 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:17:37 smithi195 ceph-mon[49435]: from='client.? 172.21.15.89:0/2510119759' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T04:17:37.369 INFO:teuthology.orchestra.run.smithi089.stdout: 5 200243 9.71 MB/sec execute 376 sec latency 230.836 ms 2024-09-21T04:17:38.070 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:17:38 smithi089 ceph-mon[64076]: from='client.34298 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T04:17:38.070 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:17:38 smithi089 ceph-mon[64076]: from='client.? 172.21.15.89:0/349436386' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-21T04:17:38.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:17:38 smithi195 ceph-mon[49435]: from='client.34298 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T04:17:38.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:17:38 smithi195 ceph-mon[49435]: from='client.? 172.21.15.89:0/349436386' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-21T04:17:38.369 INFO:teuthology.orchestra.run.smithi089.stdout: 5 200688 9.72 MB/sec execute 377 sec latency 51.051 ms 2024-09-21T04:17:39.096 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:17:39 smithi089 ceph-mon[64076]: pgmap v132: 97 pgs: 6 active+undersized, 20 active+undersized+degraded, 71 active+clean; 331 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 5.1 KiB/s rd, 4.4 MiB/s wr, 468 op/s; 149/1626 objects degraded (9.164%) 2024-09-21T04:17:39.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:17:39 smithi195 ceph-mon[49435]: pgmap v132: 97 pgs: 6 active+undersized, 20 active+undersized+degraded, 71 active+clean; 331 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 5.1 KiB/s rd, 4.4 MiB/s wr, 468 op/s; 149/1626 objects degraded (9.164%) 2024-09-21T04:17:39.369 INFO:teuthology.orchestra.run.smithi089.stdout: 5 201030 9.71 MB/sec execute 378 sec latency 77.320 ms 2024-09-21T04:17:40.103 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:17:40 smithi089 ceph-mon[64076]: Health check update: Degraded data redundancy: 149/1626 objects degraded (9.164%), 20 pgs degraded (PG_DEGRADED) 2024-09-21T04:17:40.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:17:40 smithi195 ceph-mon[49435]: Health check update: Degraded data redundancy: 149/1626 objects degraded (9.164%), 20 pgs degraded (PG_DEGRADED) 2024-09-21T04:17:40.369 INFO:teuthology.orchestra.run.smithi089.stdout: 5 201410 9.71 MB/sec execute 379 sec latency 53.202 ms 2024-09-21T04:17:41.121 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:17:41 smithi089 ceph-mon[64076]: pgmap v133: 97 pgs: 6 active+undersized, 20 active+undersized+degraded, 71 active+clean; 326 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 8.4 KiB/s rd, 4.0 MiB/s wr, 367 op/s; 147/1587 objects degraded (9.263%) 2024-09-21T04:17:41.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:17:41 smithi195 ceph-mon[49435]: pgmap v133: 97 pgs: 6 active+undersized, 20 active+undersized+degraded, 71 active+clean; 326 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 8.4 KiB/s rd, 4.0 MiB/s wr, 367 op/s; 147/1587 objects degraded (9.263%) 2024-09-21T04:17:41.369 INFO:teuthology.orchestra.run.smithi089.stdout: 5 201679 9.70 MB/sec execute 380 sec latency 57.660 ms 2024-09-21T04:17:42.369 INFO:teuthology.orchestra.run.smithi089.stdout: 5 202051 9.72 MB/sec execute 381 sec latency 45.321 ms 2024-09-21T04:17:43.175 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:17:43 smithi089 ceph-mon[64076]: pgmap v134: 97 pgs: 6 active+undersized, 20 active+undersized+degraded, 71 active+clean; 326 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 5.2 KiB/s rd, 3.4 MiB/s wr, 327 op/s; 147/1587 objects degraded (9.263%) 2024-09-21T04:17:43.175 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:17:43 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:17:43.175 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:17:43 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:17:43.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:17:43 smithi195 ceph-mon[49435]: pgmap v134: 97 pgs: 6 active+undersized, 20 active+undersized+degraded, 71 active+clean; 326 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 5.2 KiB/s rd, 3.4 MiB/s wr, 327 op/s; 147/1587 objects degraded (9.263%) 2024-09-21T04:17:43.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:17:43 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:17:43.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:17:43 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:17:43.369 INFO:teuthology.orchestra.run.smithi089.stdout: 5 202343 9.71 MB/sec execute 382 sec latency 43.283 ms 2024-09-21T04:17:43.687 INFO:journalctl@ceph.osd.2.smithi089.stdout:Sep 21 04:17:43 smithi089 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-2[73905]: 2024-09-21T04:17:43.658+0000 7f4f1ee52640 -1 osd.2 0 read_superblock omap replica is missing. 2024-09-21T04:17:44.369 INFO:teuthology.orchestra.run.smithi089.stdout: 5 202703 9.73 MB/sec execute 383 sec latency 37.600 ms 2024-09-21T04:17:44.446 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:17:44 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:17:44.446 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:17:44 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:17:44.446 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:17:44 smithi089 ceph-mon[64076]: pgmap v135: 97 pgs: 6 active+undersized, 20 active+undersized+degraded, 71 active+clean; 332 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 7.8 KiB/s rd, 5.7 MiB/s wr, 533 op/s; 148/1605 objects degraded (9.221%) 2024-09-21T04:17:44.697 INFO:journalctl@ceph.osd.2.smithi089.stdout:Sep 21 04:17:44 smithi089 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-2[73905]: 2024-09-21T04:17:44.634+0000 7f4f1ee52640 -1 osd.2 93 log_to_monitors true 2024-09-21T04:17:44.859 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:17:44 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:17:44.859 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:17:44 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:17:44.859 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:17:44 smithi195 ceph-mon[49435]: pgmap v135: 97 pgs: 6 active+undersized, 20 active+undersized+degraded, 71 active+clean; 332 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 7.8 KiB/s rd, 5.7 MiB/s wr, 533 op/s; 148/1605 objects degraded (9.221%) 2024-09-21T04:17:45.369 INFO:teuthology.orchestra.run.smithi089.stdout: 5 202995 9.71 MB/sec execute 384 sec latency 42.642 ms 2024-09-21T04:17:45.468 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:17:45 smithi089 ceph-mon[64076]: from='osd.2 [v2:172.21.15.89:6818/420208998,v1:172.21.15.89:6819/420208998]' entity='osd.2' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]} : dispatch 2024-09-21T04:17:45.468 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:17:45 smithi089 ceph-mon[64076]: Health check update: Degraded data redundancy: 148/1605 objects degraded (9.221%), 20 pgs degraded (PG_DEGRADED) 2024-09-21T04:17:45.859 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:17:45 smithi195 ceph-mon[49435]: from='osd.2 [v2:172.21.15.89:6818/420208998,v1:172.21.15.89:6819/420208998]' entity='osd.2' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]} : dispatch 2024-09-21T04:17:45.859 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:17:45 smithi195 ceph-mon[49435]: Health check update: Degraded data redundancy: 148/1605 objects degraded (9.221%), 20 pgs degraded (PG_DEGRADED) 2024-09-21T04:17:46.237 INFO:journalctl@ceph.osd.2.smithi089.stdout:Sep 21 04:17:45 smithi089 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-2[73905]: 2024-09-21T04:17:45.976+0000 7f4f12bf4640 -1 osd.2 93 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-21T04:17:46.369 INFO:teuthology.orchestra.run.smithi089.stdout: 5 203269 9.70 MB/sec execute 385 sec latency 152.419 ms 2024-09-21T04:17:46.488 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:17:46 smithi089 ceph-mon[64076]: from='osd.2 [v2:172.21.15.89:6818/420208998,v1:172.21.15.89:6819/420208998]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-09-21T04:17:46.488 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:17:46 smithi089 ceph-mon[64076]: osdmap e96: 8 total, 7 up, 8 in 2024-09-21T04:17:46.488 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:17:46 smithi089 ceph-mon[64076]: from='osd.2 [v2:172.21.15.89:6818/420208998,v1:172.21.15.89:6819/420208998]' entity='osd.2' cmd={"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi089", "root=default"]} : dispatch 2024-09-21T04:17:46.488 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:17:46 smithi089 ceph-mon[64076]: from='osd.2 [v2:172.21.15.89:6818/420208998,v1:172.21.15.89:6819/420208998]' entity='osd.2' 2024-09-21T04:17:46.488 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:17:46 smithi089 ceph-mon[64076]: pgmap v137: 97 pgs: 6 active+undersized, 20 active+undersized+degraded, 71 active+clean; 332 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 7.1 KiB/s rd, 4.7 MiB/s wr, 463 op/s; 148/1605 objects degraded (9.221%) 2024-09-21T04:17:46.859 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:17:46 smithi195 ceph-mon[49435]: from='osd.2 [v2:172.21.15.89:6818/420208998,v1:172.21.15.89:6819/420208998]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-09-21T04:17:46.859 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:17:46 smithi195 ceph-mon[49435]: osdmap e96: 8 total, 7 up, 8 in 2024-09-21T04:17:46.859 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:17:46 smithi195 ceph-mon[49435]: from='osd.2 [v2:172.21.15.89:6818/420208998,v1:172.21.15.89:6819/420208998]' entity='osd.2' cmd={"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi089", "root=default"]} : dispatch 2024-09-21T04:17:46.859 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:17:46 smithi195 ceph-mon[49435]: from='osd.2 [v2:172.21.15.89:6818/420208998,v1:172.21.15.89:6819/420208998]' entity='osd.2' 2024-09-21T04:17:46.859 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:17:46 smithi195 ceph-mon[49435]: pgmap v137: 97 pgs: 6 active+undersized, 20 active+undersized+degraded, 71 active+clean; 332 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 7.1 KiB/s rd, 4.7 MiB/s wr, 463 op/s; 148/1605 objects degraded (9.221%) 2024-09-21T04:17:47.369 INFO:teuthology.orchestra.run.smithi089.stdout: 5 203473 9.68 MB/sec execute 386 sec latency 447.427 ms 2024-09-21T04:17:47.765 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:17:47 smithi089 ceph-mon[64076]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-21T04:17:47.765 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:17:47 smithi089 ceph-mon[64076]: osd.2 [v2:172.21.15.89:6818/420208998,v1:172.21.15.89:6819/420208998] boot 2024-09-21T04:17:47.765 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:17:47 smithi089 ceph-mon[64076]: osdmap e97: 8 total, 8 up, 8 in 2024-09-21T04:17:47.765 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:17:47 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "osd metadata", "id": 2} : dispatch 2024-09-21T04:17:47.765 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:17:47 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:17:47.765 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:17:47 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T04:17:47.859 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:17:47 smithi195 ceph-mon[49435]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-21T04:17:47.859 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:17:47 smithi195 ceph-mon[49435]: osd.2 [v2:172.21.15.89:6818/420208998,v1:172.21.15.89:6819/420208998] boot 2024-09-21T04:17:47.859 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:17:47 smithi195 ceph-mon[49435]: osdmap e97: 8 total, 8 up, 8 in 2024-09-21T04:17:47.859 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:17:47 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "osd metadata", "id": 2} : dispatch 2024-09-21T04:17:47.859 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:17:47 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:17:47.859 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:17:47 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T04:17:48.369 INFO:teuthology.orchestra.run.smithi089.stdout: 5 203473 9.65 MB/sec execute 387 sec latency 1447.478 ms 2024-09-21T04:17:49.132 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:17:48 smithi089 ceph-mon[64076]: osdmap e98: 8 total, 8 up, 8 in 2024-09-21T04:17:49.132 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:17:48 smithi089 ceph-mon[64076]: pgmap v140: 97 pgs: 1 activating+degraded, 1 activating, 3 remapped+peering, 6 active+undersized, 13 active+undersized+degraded, 73 active+clean; 331 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 5.2 KiB/s rd, 5.9 MiB/s wr, 564 op/s; 103/1629 objects degraded (6.323%) 2024-09-21T04:17:49.132 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:17:48 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:17:49.132 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:17:48 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:17:49.132 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:17:48 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T04:17:49.132 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:17:48 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T04:17:49.132 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:17:48 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:17:49.132 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:17:48 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:17:49.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:17:48 smithi195 ceph-mon[49435]: osdmap e98: 8 total, 8 up, 8 in 2024-09-21T04:17:49.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:17:48 smithi195 ceph-mon[49435]: pgmap v140: 97 pgs: 1 activating+degraded, 1 activating, 3 remapped+peering, 6 active+undersized, 13 active+undersized+degraded, 73 active+clean; 331 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 5.2 KiB/s rd, 5.9 MiB/s wr, 564 op/s; 103/1629 objects degraded (6.323%) 2024-09-21T04:17:49.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:17:48 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:17:49.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:17:48 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:17:49.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:17:48 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T04:17:49.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:17:48 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T04:17:49.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:17:48 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:17:49.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:17:48 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:17:49.369 INFO:teuthology.orchestra.run.smithi089.stdout: 5 203560 9.64 MB/sec execute 388 sec latency 2206.952 ms 2024-09-21T04:17:50.264 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:17:49 smithi089 ceph-mon[64076]: osdmap e99: 8 total, 8 up, 8 in 2024-09-21T04:17:50.265 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:17:49 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T04:17:50.265 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:17:49 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "versions"} : dispatch 2024-09-21T04:17:50.265 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:17:49 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "versions"} : dispatch 2024-09-21T04:17:50.265 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:17:49 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "versions"} : dispatch 2024-09-21T04:17:50.265 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:17:49 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16} : dispatch 2024-09-21T04:17:50.265 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:17:49 smithi089 ceph-mon[64076]: Health check update: Degraded data redundancy: 103/1629 objects degraded (6.323%), 14 pgs degraded (PG_DEGRADED) 2024-09-21T04:17:50.265 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:17:49 smithi089 ceph-mon[64076]: osdmap e100: 8 total, 8 up, 8 in 2024-09-21T04:17:50.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:17:49 smithi195 ceph-mon[49435]: osdmap e99: 8 total, 8 up, 8 in 2024-09-21T04:17:50.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:17:49 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T04:17:50.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:17:49 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "versions"} : dispatch 2024-09-21T04:17:50.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:17:49 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "versions"} : dispatch 2024-09-21T04:17:50.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:17:49 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "versions"} : dispatch 2024-09-21T04:17:50.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:17:49 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16} : dispatch 2024-09-21T04:17:50.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:17:49 smithi195 ceph-mon[49435]: Health check update: Degraded data redundancy: 103/1629 objects degraded (6.323%), 14 pgs degraded (PG_DEGRADED) 2024-09-21T04:17:50.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:17:49 smithi195 ceph-mon[49435]: osdmap e100: 8 total, 8 up, 8 in 2024-09-21T04:17:50.369 INFO:teuthology.orchestra.run.smithi089.stdout: 5 203680 9.63 MB/sec execute 389 sec latency 499.857 ms 2024-09-21T04:17:51.153 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:17:50 smithi089 ceph-mon[64076]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-09-21T04:17:51.153 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:17:50 smithi089 ceph-mon[64076]: Upgrade: unsafe to stop osd(s) at this time (5 PGs are or would become offline) 2024-09-21T04:17:51.153 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:17:50 smithi089 ceph-mon[64076]: pgmap v143: 97 pgs: 1 activating+degraded, 1 activating, 9 remapped+peering, 1 active+undersized, 5 active+undersized+degraded, 80 active+clean; 333 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 511 B/s rd, 4.1 MiB/s wr, 461 op/s; 29/1620 objects degraded (1.790%) 2024-09-21T04:17:51.153 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:17:50 smithi089 ceph-mon[64076]: osdmap e101: 8 total, 8 up, 8 in 2024-09-21T04:17:51.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:17:50 smithi195 ceph-mon[49435]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-09-21T04:17:51.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:17:50 smithi195 ceph-mon[49435]: Upgrade: unsafe to stop osd(s) at this time (5 PGs are or would become offline) 2024-09-21T04:17:51.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:17:50 smithi195 ceph-mon[49435]: pgmap v143: 97 pgs: 1 activating+degraded, 1 activating, 9 remapped+peering, 1 active+undersized, 5 active+undersized+degraded, 80 active+clean; 333 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 511 B/s rd, 4.1 MiB/s wr, 461 op/s; 29/1620 objects degraded (1.790%) 2024-09-21T04:17:51.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:17:50 smithi195 ceph-mon[49435]: osdmap e101: 8 total, 8 up, 8 in 2024-09-21T04:17:51.369 INFO:teuthology.orchestra.run.smithi089.stdout: 5 203812 9.61 MB/sec execute 390 sec latency 1030.900 ms 2024-09-21T04:17:52.369 INFO:teuthology.orchestra.run.smithi089.stdout: 5 204327 9.62 MB/sec execute 391 sec latency 375.389 ms 2024-09-21T04:17:53.178 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:17:52 smithi089 ceph-mon[64076]: osdmap e102: 8 total, 8 up, 8 in 2024-09-21T04:17:53.178 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:17:52 smithi089 ceph-mon[64076]: pgmap v146: 97 pgs: 1 activating+degraded, 1 activating, 9 remapped+peering, 86 active+clean; 333 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 2.1 MiB/s wr, 239 op/s; 2/1620 objects degraded (0.123%); 30 KiB/s, 0 objects/s recovering 2024-09-21T04:17:53.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:17:52 smithi195 ceph-mon[49435]: osdmap e102: 8 total, 8 up, 8 in 2024-09-21T04:17:53.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:17:52 smithi195 ceph-mon[49435]: pgmap v146: 97 pgs: 1 activating+degraded, 1 activating, 9 remapped+peering, 86 active+clean; 333 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 2.1 MiB/s wr, 239 op/s; 2/1620 objects degraded (0.123%); 30 KiB/s, 0 objects/s recovering 2024-09-21T04:17:53.369 INFO:teuthology.orchestra.run.smithi089.stdout: 5 204586 9.61 MB/sec execute 392 sec latency 975.714 ms 2024-09-21T04:17:53.991 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:17:53 smithi089 ceph-mon[64076]: osdmap e103: 8 total, 8 up, 8 in 2024-09-21T04:17:54.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:17:53 smithi195 ceph-mon[49435]: osdmap e103: 8 total, 8 up, 8 in 2024-09-21T04:17:54.369 INFO:teuthology.orchestra.run.smithi089.stdout: 5 204749 9.59 MB/sec execute 393 sec latency 941.256 ms 2024-09-21T04:17:55.015 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:17:54 smithi089 ceph-mon[64076]: osdmap e104: 8 total, 8 up, 8 in 2024-09-21T04:17:55.015 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:17:54 smithi089 ceph-mon[64076]: pgmap v149: 97 pgs: 2 active+undersized+remapped, 2 peering, 93 active+clean; 326 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 9.5 KiB/s rd, 3.0 MiB/s wr, 363 op/s; 1.1 MiB/s, 52 objects/s recovering 2024-09-21T04:17:55.015 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:17:54 smithi089 ceph-mon[64076]: Health check update: Degraded data redundancy: 2/1620 objects degraded (0.123%), 1 pg degraded (PG_DEGRADED) 2024-09-21T04:17:55.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:17:54 smithi195 ceph-mon[49435]: osdmap e104: 8 total, 8 up, 8 in 2024-09-21T04:17:55.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:17:54 smithi195 ceph-mon[49435]: pgmap v149: 97 pgs: 2 active+undersized+remapped, 2 peering, 93 active+clean; 326 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 9.5 KiB/s rd, 3.0 MiB/s wr, 363 op/s; 1.1 MiB/s, 52 objects/s recovering 2024-09-21T04:17:55.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:17:54 smithi195 ceph-mon[49435]: Health check update: Degraded data redundancy: 2/1620 objects degraded (0.123%), 1 pg degraded (PG_DEGRADED) 2024-09-21T04:17:55.369 INFO:teuthology.orchestra.run.smithi089.stdout: 5 205026 9.58 MB/sec execute 394 sec latency 72.597 ms 2024-09-21T04:17:56.041 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:17:55 smithi089 ceph-mon[64076]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/1620 objects degraded (0.123%), 1 pg degraded) 2024-09-21T04:17:56.041 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:17:55 smithi089 ceph-mon[64076]: Cluster is now healthy 2024-09-21T04:17:56.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:17:55 smithi195 ceph-mon[49435]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/1620 objects degraded (0.123%), 1 pg degraded) 2024-09-21T04:17:56.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:17:55 smithi195 ceph-mon[49435]: Cluster is now healthy 2024-09-21T04:17:56.369 INFO:teuthology.orchestra.run.smithi089.stdout: 5 205369 9.59 MB/sec execute 395 sec latency 53.840 ms 2024-09-21T04:17:57.048 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:17:57 smithi089 ceph-mon[64076]: pgmap v150: 97 pgs: 2 active+undersized+remapped, 2 peering, 93 active+clean; 326 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 2.9 KiB/s rd, 2.3 MiB/s wr, 284 op/s; 897 KiB/s, 41 objects/s recovering 2024-09-21T04:17:57.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:17:57 smithi195 ceph-mon[49435]: pgmap v150: 97 pgs: 2 active+undersized+remapped, 2 peering, 93 active+clean; 326 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 2.9 KiB/s rd, 2.3 MiB/s wr, 284 op/s; 897 KiB/s, 41 objects/s recovering 2024-09-21T04:17:57.369 INFO:teuthology.orchestra.run.smithi089.stdout: 5 205663 9.59 MB/sec execute 396 sec latency 43.628 ms 2024-09-21T04:17:58.370 INFO:teuthology.orchestra.run.smithi089.stdout: 5 205972 9.61 MB/sec execute 397 sec latency 58.946 ms 2024-09-21T04:17:59.133 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:17:59 smithi089 ceph-mon[64076]: pgmap v151: 97 pgs: 2 peering, 95 active+clean; 335 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 2.5 KiB/s rd, 4.6 MiB/s wr, 496 op/s; 757 KiB/s, 35 objects/s recovering 2024-09-21T04:17:59.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:17:59 smithi195 ceph-mon[49435]: pgmap v151: 97 pgs: 2 peering, 95 active+clean; 335 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 2.5 KiB/s rd, 4.6 MiB/s wr, 496 op/s; 757 KiB/s, 35 objects/s recovering 2024-09-21T04:17:59.369 INFO:teuthology.orchestra.run.smithi089.stdout: 5 206235 9.60 MB/sec execute 398 sec latency 65.324 ms 2024-09-21T04:18:00.369 INFO:teuthology.orchestra.run.smithi089.stdout: 5 206481 9.58 MB/sec execute 399 sec latency 65.060 ms 2024-09-21T04:18:01.199 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:18:01 smithi089 ceph-mon[64076]: pgmap v152: 97 pgs: 97 active+clean; 332 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 2.1 KiB/s rd, 4.5 MiB/s wr, 424 op/s; 570 KiB/s, 26 objects/s recovering 2024-09-21T04:18:01.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:18:01 smithi195 ceph-mon[49435]: pgmap v152: 97 pgs: 97 active+clean; 332 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 2.1 KiB/s rd, 4.5 MiB/s wr, 424 op/s; 570 KiB/s, 26 objects/s recovering 2024-09-21T04:18:01.369 INFO:teuthology.orchestra.run.smithi089.stdout: 5 206629 9.57 MB/sec execute 400 sec latency 82.622 ms 2024-09-21T04:18:02.370 INFO:teuthology.orchestra.run.smithi089.stdout: 5 206814 9.55 MB/sec execute 401 sec latency 77.115 ms 2024-09-21T04:18:03.029 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:18:03 smithi089 ceph-mon[64076]: pgmap v153: 97 pgs: 97 active+clean; 332 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 4.8 KiB/s rd, 4.0 MiB/s wr, 376 op/s; 506 KiB/s, 23 objects/s recovering 2024-09-21T04:18:03.029 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:18:03 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:18:03.029 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:18:03 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T04:18:03.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:18:03 smithi195 ceph-mon[49435]: pgmap v153: 97 pgs: 97 active+clean; 332 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 4.8 KiB/s rd, 4.0 MiB/s wr, 376 op/s; 506 KiB/s, 23 objects/s recovering 2024-09-21T04:18:03.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:18:03 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:18:03.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:18:03 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T04:18:03.370 INFO:teuthology.orchestra.run.smithi089.stdout: 5 207029 9.54 MB/sec execute 402 sec latency 72.231 ms 2024-09-21T04:18:04.370 INFO:teuthology.orchestra.run.smithi089.stdout: 5 207304 9.54 MB/sec execute 403 sec latency 63.507 ms 2024-09-21T04:18:05.067 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:18:05 smithi089 ceph-mon[64076]: pgmap v154: 97 pgs: 97 active+clean; 334 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 4.3 KiB/s rd, 4.1 MiB/s wr, 368 op/s 2024-09-21T04:18:05.067 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:18:05 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16} : dispatch 2024-09-21T04:18:05.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:18:05 smithi195 ceph-mon[49435]: pgmap v154: 97 pgs: 97 active+clean; 334 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 4.3 KiB/s rd, 4.1 MiB/s wr, 368 op/s 2024-09-21T04:18:05.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:18:05 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16} : dispatch 2024-09-21T04:18:05.370 INFO:teuthology.orchestra.run.smithi089.stdout: 5 207635 9.54 MB/sec execute 404 sec latency 67.735 ms 2024-09-21T04:18:06.120 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:18:06 smithi089 ceph-mon[64076]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-09-21T04:18:06.120 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:18:06 smithi089 ceph-mon[64076]: Upgrade: osd.3 is safe to restart 2024-09-21T04:18:06.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:18:06 smithi195 ceph-mon[49435]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-09-21T04:18:06.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:18:06 smithi195 ceph-mon[49435]: Upgrade: osd.3 is safe to restart 2024-09-21T04:18:06.370 INFO:teuthology.orchestra.run.smithi089.stdout: 5 207982 9.53 MB/sec execute 405 sec latency 65.227 ms 2024-09-21T04:18:07.139 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:18:07 smithi089 ceph-mon[64076]: pgmap v155: 97 pgs: 97 active+clean; 334 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.6 KiB/s rd, 3.4 MiB/s wr, 309 op/s 2024-09-21T04:18:07.139 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:18:07 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:18:07.139 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:18:07 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "auth get", "entity": "osd.3"} : dispatch 2024-09-21T04:18:07.139 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:18:07 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T04:18:07.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:18:07 smithi195 ceph-mon[49435]: pgmap v155: 97 pgs: 97 active+clean; 334 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.6 KiB/s rd, 3.4 MiB/s wr, 309 op/s 2024-09-21T04:18:07.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:18:07 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:18:07.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:18:07 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "auth get", "entity": "osd.3"} : dispatch 2024-09-21T04:18:07.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:18:07 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T04:18:07.370 INFO:teuthology.orchestra.run.smithi089.stdout: 5 208263 9.52 MB/sec execute 406 sec latency 65.874 ms 2024-09-21T04:18:07.688 INFO:teuthology.orchestra.run.smithi089.stdout:true 2024-09-21T04:18:08.147 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:18:08 smithi089 ceph-mon[64076]: Upgrade: Updating osd.3 2024-09-21T04:18:08.147 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:18:08 smithi089 ceph-mon[64076]: Deploying daemon osd.3 on smithi089 2024-09-21T04:18:08.147 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:18:08 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:18:08.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:18:08 smithi195 ceph-mon[49435]: Upgrade: Updating osd.3 2024-09-21T04:18:08.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:18:08 smithi195 ceph-mon[49435]: Deploying daemon osd.3 on smithi089 2024-09-21T04:18:08.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:18:08 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:18:08.370 INFO:teuthology.orchestra.run.smithi089.stdout: 5 208523 9.51 MB/sec execute 407 sec latency 57.991 ms 2024-09-21T04:18:08.695 INFO:teuthology.orchestra.run.smithi089.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-21T04:18:08.695 INFO:teuthology.orchestra.run.smithi089.stdout:alertmanager.smithi089 smithi089 *:9093,9094 running (6m) 26s ago 15m 17.4M - 0.27.0 11f11916f8cd 65dfcbcafee1 2024-09-21T04:18:08.695 INFO:teuthology.orchestra.run.smithi089.stdout:crash.smithi089 smithi089 running (2m) 26s ago 15m 6891k - 19.3.0-5103-gf36283e7 48ed0a2cf89d 6c415a5e173e 2024-09-21T04:18:08.695 INFO:teuthology.orchestra.run.smithi089.stdout:crash.smithi195 smithi195 running (2m) 2m ago 14m 6891k - 19.3.0-5103-gf36283e7 48ed0a2cf89d 78ce05a4e9ab 2024-09-21T04:18:08.695 INFO:teuthology.orchestra.run.smithi089.stdout:grafana.smithi089 smithi089 *:3000 running (6m) 26s ago 15m 45.9M - dad864ee21e9 ec4d6738bce7 2024-09-21T04:18:08.695 INFO:teuthology.orchestra.run.smithi089.stdout:mds.foofs.smithi089.osufws smithi089 running (9m) 26s ago 9m 38.3M - 17.2.0 e1d6a67b021e 664d41859bea 2024-09-21T04:18:08.696 INFO:teuthology.orchestra.run.smithi089.stdout:mds.foofs.smithi195.hijxre smithi195 running (9m) 2m ago 9m 16.7M - 17.2.0 e1d6a67b021e 7bc0c6542d12 2024-09-21T04:18:08.696 INFO:teuthology.orchestra.run.smithi089.stdout:mgr.smithi089.srlyxf smithi089 *:8443,9283,8765 running (5m) 26s ago 16m 517M - 19.3.0-5103-gf36283e7 48ed0a2cf89d 3cbeaaa88241 2024-09-21T04:18:08.696 INFO:teuthology.orchestra.run.smithi089.stdout:mgr.smithi195.cgotzm smithi195 *:8443,9283,8765 running (4m) 2m ago 14m 454M - 19.3.0-5103-gf36283e7 48ed0a2cf89d d3734574cf66 2024-09-21T04:18:08.696 INFO:teuthology.orchestra.run.smithi089.stdout:mon.smithi089 smithi089 running (4m) 26s ago 16m 56.7M 2048M 19.3.0-5103-gf36283e7 48ed0a2cf89d 7dd34c8a6f21 2024-09-21T04:18:08.696 INFO:teuthology.orchestra.run.smithi089.stdout:mon.smithi195 smithi195 running (3m) 2m ago 14m 36.7M 2048M 19.3.0-5103-gf36283e7 48ed0a2cf89d 4255937444e7 2024-09-21T04:18:08.696 INFO:teuthology.orchestra.run.smithi089.stdout:nfs.foo.0.0.smithi089.dcjmqc smithi089 *:2049 running (9m) 26s ago 9m 282M - 4.0 e1d6a67b021e 98550f94e773 2024-09-21T04:18:08.696 INFO:teuthology.orchestra.run.smithi089.stdout:nfs.foo.1.0.smithi195.vhoayp smithi195 *:2049 running (8m) 2m ago 8m 88.0M - 4.0 e1d6a67b021e c60a4040130a 2024-09-21T04:18:08.696 INFO:teuthology.orchestra.run.smithi089.stdout:node-exporter.smithi089 smithi089 *:9100 running (6m) 26s ago 15m 10.2M - 1.7.0 72c9c2088986 7ef0d1588475 2024-09-21T04:18:08.696 INFO:teuthology.orchestra.run.smithi089.stdout:node-exporter.smithi195 smithi195 *:9100 running (5m) 2m ago 14m 10.7M - 1.7.0 72c9c2088986 61eafd61bc6c 2024-09-21T04:18:08.696 INFO:teuthology.orchestra.run.smithi089.stdout:osd.0 smithi089 running (2m) 26s ago 13m 300M 1935M 19.3.0-5103-gf36283e7 48ed0a2cf89d 3df0adea59fb 2024-09-21T04:18:08.696 INFO:teuthology.orchestra.run.smithi089.stdout:osd.1 smithi089 running (86s) 26s ago 13m 225M 1935M 19.3.0-5103-gf36283e7 48ed0a2cf89d da2e49774e7e 2024-09-21T04:18:08.696 INFO:teuthology.orchestra.run.smithi089.stdout:osd.2 smithi089 running (35s) 26s ago 12m 11.2M 1935M 19.3.0-5103-gf36283e7 48ed0a2cf89d b9f6438dd5b9 2024-09-21T04:18:08.696 INFO:teuthology.orchestra.run.smithi089.stdout:osd.3 smithi089 running (12m) 26s ago 12m 354M 1935M 17.2.0 e1d6a67b021e 31c610ed5815 2024-09-21T04:18:08.696 INFO:teuthology.orchestra.run.smithi089.stdout:osd.4 smithi195 running (12m) 2m ago 12m 524M 2704M 17.2.0 e1d6a67b021e f00e18a1f220 2024-09-21T04:18:08.696 INFO:teuthology.orchestra.run.smithi089.stdout:osd.5 smithi195 running (11m) 2m ago 11m 318M 2704M 17.2.0 e1d6a67b021e cecfd6ff1eee 2024-09-21T04:18:08.696 INFO:teuthology.orchestra.run.smithi089.stdout:osd.6 smithi195 running (11m) 2m ago 11m 510M 2704M 17.2.0 e1d6a67b021e 0d7db362cc23 2024-09-21T04:18:08.697 INFO:teuthology.orchestra.run.smithi089.stdout:osd.7 smithi195 running (10m) 2m ago 10m 387M 2704M 17.2.0 e1d6a67b021e 0109b3fea188 2024-09-21T04:18:08.697 INFO:teuthology.orchestra.run.smithi089.stdout:prometheus.smithi089 smithi089 *:9095 running (4m) 26s ago 14m 53.2M - 2.51.0 1d3b7f56885b 8c9f309a6459 2024-09-21T04:18:09.159 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:18:09 smithi089 ceph-mon[64076]: from='client.34308 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T04:18:09.160 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:18:09 smithi089 ceph-mon[64076]: pgmap v156: 97 pgs: 97 active+clean; 330 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 5.3 KiB/s rd, 3.9 MiB/s wr, 348 op/s 2024-09-21T04:18:09.307 INFO:teuthology.orchestra.run.smithi089.stdout:{ 2024-09-21T04:18:09.307 INFO:teuthology.orchestra.run.smithi089.stdout: "mon": { 2024-09-21T04:18:09.307 INFO:teuthology.orchestra.run.smithi089.stdout: "ceph version 19.3.0-5103-gf36283e7 (f36283e7c91cac1c01cb9a7c4ff21bd697dbd4f2) squid (dev)": 2 2024-09-21T04:18:09.307 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2024-09-21T04:18:09.307 INFO:teuthology.orchestra.run.smithi089.stdout: "mgr": { 2024-09-21T04:18:09.307 INFO:teuthology.orchestra.run.smithi089.stdout: "ceph version 19.3.0-5103-gf36283e7 (f36283e7c91cac1c01cb9a7c4ff21bd697dbd4f2) squid (dev)": 2 2024-09-21T04:18:09.307 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2024-09-21T04:18:09.307 INFO:teuthology.orchestra.run.smithi089.stdout: "osd": { 2024-09-21T04:18:09.307 INFO:teuthology.orchestra.run.smithi089.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 5, 2024-09-21T04:18:09.307 INFO:teuthology.orchestra.run.smithi089.stdout: "ceph version 19.3.0-5103-gf36283e7 (f36283e7c91cac1c01cb9a7c4ff21bd697dbd4f2) squid (dev)": 3 2024-09-21T04:18:09.307 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2024-09-21T04:18:09.308 INFO:teuthology.orchestra.run.smithi089.stdout: "mds": { 2024-09-21T04:18:09.308 INFO:teuthology.orchestra.run.smithi089.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-21T04:18:09.308 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2024-09-21T04:18:09.308 INFO:teuthology.orchestra.run.smithi089.stdout: "overall": { 2024-09-21T04:18:09.308 INFO:teuthology.orchestra.run.smithi089.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 7, 2024-09-21T04:18:09.308 INFO:teuthology.orchestra.run.smithi089.stdout: "ceph version 19.3.0-5103-gf36283e7 (f36283e7c91cac1c01cb9a7c4ff21bd697dbd4f2) squid (dev)": 7 2024-09-21T04:18:09.308 INFO:teuthology.orchestra.run.smithi089.stdout: } 2024-09-21T04:18:09.308 INFO:teuthology.orchestra.run.smithi089.stdout:} 2024-09-21T04:18:09.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:18:09 smithi195 ceph-mon[49435]: from='client.34308 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T04:18:09.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:18:09 smithi195 ceph-mon[49435]: pgmap v156: 97 pgs: 97 active+clean; 330 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 5.3 KiB/s rd, 3.9 MiB/s wr, 348 op/s 2024-09-21T04:18:09.370 INFO:teuthology.orchestra.run.smithi089.stdout: 5 208833 9.51 MB/sec execute 408 sec latency 55.110 ms 2024-09-21T04:18:09.806 INFO:teuthology.orchestra.run.smithi089.stdout:{ 2024-09-21T04:18:09.806 INFO:teuthology.orchestra.run.smithi089.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:2195e5f178be0a9390728e2a14088236d8fe6f82635f5c84418fe37a04457e00", 2024-09-21T04:18:09.807 INFO:teuthology.orchestra.run.smithi089.stdout: "in_progress": true, 2024-09-21T04:18:09.807 INFO:teuthology.orchestra.run.smithi089.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-21T04:18:09.807 INFO:teuthology.orchestra.run.smithi089.stdout: "services_complete": [ 2024-09-21T04:18:09.807 INFO:teuthology.orchestra.run.smithi089.stdout: "mgr", 2024-09-21T04:18:09.807 INFO:teuthology.orchestra.run.smithi089.stdout: "crash", 2024-09-21T04:18:09.807 INFO:teuthology.orchestra.run.smithi089.stdout: "mon" 2024-09-21T04:18:09.807 INFO:teuthology.orchestra.run.smithi089.stdout: ], 2024-09-21T04:18:09.807 INFO:teuthology.orchestra.run.smithi089.stdout: "progress": "9/23 daemons upgraded", 2024-09-21T04:18:09.807 INFO:teuthology.orchestra.run.smithi089.stdout: "message": "Currently upgrading osd daemons", 2024-09-21T04:18:09.807 INFO:teuthology.orchestra.run.smithi089.stdout: "is_paused": false 2024-09-21T04:18:09.807 INFO:teuthology.orchestra.run.smithi089.stdout:} 2024-09-21T04:18:10.205 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:18:10 smithi089 ceph-mon[64076]: from='client.34312 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T04:18:10.206 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:18:10 smithi089 ceph-mon[64076]: from='client.44223 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T04:18:10.206 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:18:10 smithi089 ceph-mon[64076]: from='client.? 172.21.15.89:0/3902685717' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T04:18:10.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:18:10 smithi195 ceph-mon[49435]: from='client.34312 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T04:18:10.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:18:10 smithi195 ceph-mon[49435]: from='client.44223 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T04:18:10.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:18:10 smithi195 ceph-mon[49435]: from='client.? 172.21.15.89:0/3902685717' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T04:18:10.370 INFO:teuthology.orchestra.run.smithi089.stdout: 5 209187 9.52 MB/sec execute 409 sec latency 59.842 ms 2024-09-21T04:18:10.536 INFO:teuthology.orchestra.run.smithi089.stdout:HEALTH_OK 2024-09-21T04:18:11.236 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:18:11 smithi089 ceph-mon[64076]: from='client.34324 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T04:18:11.236 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:18:11 smithi089 ceph-mon[64076]: pgmap v157: 97 pgs: 97 active+clean; 329 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 5.5 KiB/s rd, 3.6 MiB/s wr, 358 op/s 2024-09-21T04:18:11.236 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:18:11 smithi089 ceph-mon[64076]: from='client.? 172.21.15.89:0/1762528073' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-21T04:18:11.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:18:11 smithi195 ceph-mon[49435]: from='client.34324 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T04:18:11.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:18:11 smithi195 ceph-mon[49435]: pgmap v157: 97 pgs: 97 active+clean; 329 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 5.5 KiB/s rd, 3.6 MiB/s wr, 358 op/s 2024-09-21T04:18:11.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:18:11 smithi195 ceph-mon[49435]: from='client.? 172.21.15.89:0/1762528073' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-21T04:18:11.370 INFO:teuthology.orchestra.run.smithi089.stdout: 5 209563 9.54 MB/sec execute 410 sec latency 41.097 ms 2024-09-21T04:18:11.487 INFO:journalctl@ceph.osd.3.smithi089.stdout:Sep 21 04:18:11 smithi089 systemd[1]: Stopping Ceph osd.3 for 1b9b79fc-77ce-11ef-baf5-efdc52797490... 2024-09-21T04:18:11.994 INFO:journalctl@ceph.osd.3.smithi089.stdout:Sep 21 04:18:11 smithi089 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-3[44670]: 2024-09-21T04:18:11.919+0000 7fdb5ab96700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.3 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-21T04:18:11.994 INFO:journalctl@ceph.osd.3.smithi089.stdout:Sep 21 04:18:11 smithi089 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-3[44670]: 2024-09-21T04:18:11.919+0000 7fdb5ab96700 -1 osd.3 104 *** Got signal Terminated *** 2024-09-21T04:18:11.994 INFO:journalctl@ceph.osd.3.smithi089.stdout:Sep 21 04:18:11 smithi089 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-3[44670]: 2024-09-21T04:18:11.919+0000 7fdb5ab96700 -1 osd.3 104 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-21T04:18:12.264 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:18:12 smithi089 ceph-mon[64076]: osd.3 marked itself down and dead 2024-09-21T04:18:12.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:18:12 smithi195 ceph-mon[49435]: osd.3 marked itself down and dead 2024-09-21T04:18:12.370 INFO:teuthology.orchestra.run.smithi089.stdout: 5 209774 9.53 MB/sec execute 411 sec latency 284.591 ms 2024-09-21T04:18:12.799 INFO:journalctl@ceph.osd.3.smithi089.stdout:Sep 21 04:18:12 smithi089 podman[75934]: 2024-09-21 04:18:12.766232782 +0000 UTC m=+1.070467242 container died 31c610ed5815b49fc51fa4401ba155a4841cc053d86d8e03fecdf45b76e1f295 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-3, GIT_CLEAN=True, io.openshift.expose-services=, GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.license_terms=https://centos.org/legal/licensing-policy/, architecture=x86_64, vendor=Red Hat, Inc., com.redhat.component=centos-stream-container, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, maintainer=Guillaume Abrioux , io.k8s.display-name=CentOS Stream 8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, version=8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_BRANCH=HEAD, name=centos-stream, io.buildah.version=1.19.8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, release=754, distribution-scope=public, vcs-type=git, ceph=True, CEPH_POINT_RELEASE=-17.2.0, build-date=2022-05-03T08:36:31.336870, io.openshift.tags=base centos centos-stream, RELEASE=HEAD) 2024-09-21T04:18:13.308 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:18:13 smithi089 ceph-mon[64076]: pgmap v158: 97 pgs: 97 active+clean; 329 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.5 KiB/s rd, 2.9 MiB/s wr, 325 op/s 2024-09-21T04:18:13.308 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:18:13 smithi089 ceph-mon[64076]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-21T04:18:13.308 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:18:13 smithi089 ceph-mon[64076]: osdmap e105: 8 total, 7 up, 8 in 2024-09-21T04:18:13.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:18:13 smithi195 ceph-mon[49435]: pgmap v158: 97 pgs: 97 active+clean; 329 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.5 KiB/s rd, 2.9 MiB/s wr, 325 op/s 2024-09-21T04:18:13.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:18:13 smithi195 ceph-mon[49435]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-21T04:18:13.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:18:13 smithi195 ceph-mon[49435]: osdmap e105: 8 total, 7 up, 8 in 2024-09-21T04:18:13.370 INFO:teuthology.orchestra.run.smithi089.stdout: 5 209948 9.51 MB/sec execute 412 sec latency 1043.732 ms 2024-09-21T04:18:13.559 INFO:journalctl@ceph.osd.3.smithi089.stdout:Sep 21 04:18:13 smithi089 podman[75934]: 2024-09-21 04:18:13.363939159 +0000 UTC m=+1.668173625 container cleanup 31c610ed5815b49fc51fa4401ba155a4841cc053d86d8e03fecdf45b76e1f295 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-3, com.redhat.component=centos-stream-container, vcs-type=git, architecture=x86_64, RELEASE=HEAD, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, distribution-scope=public, vendor=Red Hat, Inc., com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_BRANCH=HEAD, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, maintainer=Guillaume Abrioux , GIT_CLEAN=True, version=8, CEPH_POINT_RELEASE=-17.2.0, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.openshift.tags=base centos centos-stream, name=centos-stream, io.buildah.version=1.19.8, io.openshift.expose-services=, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., release=754, io.k8s.display-name=CentOS Stream 8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, ceph=True, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_REPO=https://github.com/ceph/ceph-container.git, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, build-date=2022-05-03T08:36:31.336870) 2024-09-21T04:18:13.559 INFO:journalctl@ceph.osd.3.smithi089.stdout:Sep 21 04:18:13 smithi089 bash[75934]: ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-3 2024-09-21T04:18:13.812 INFO:journalctl@ceph.osd.3.smithi089.stdout:Sep 21 04:18:13 smithi089 podman[75949]: 2024-09-21 04:18:13.69858639 +0000 UTC m=+0.930952519 container remove 31c610ed5815b49fc51fa4401ba155a4841cc053d86d8e03fecdf45b76e1f295 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-3, vcs-type=git, vendor=Red Hat, Inc., io.buildah.version=1.19.8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, architecture=x86_64, build-date=2022-05-03T08:36:31.336870, com.redhat.component=centos-stream-container, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_BRANCH=HEAD, GIT_CLEAN=True, ceph=True, GIT_REPO=https://github.com/ceph/ceph-container.git, release=754, CEPH_POINT_RELEASE=-17.2.0, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.expose-services=, version=8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, name=centos-stream, maintainer=Guillaume Abrioux , RELEASE=HEAD, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, distribution-scope=public, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.k8s.display-name=CentOS Stream 8, io.openshift.tags=base centos centos-stream) 2024-09-21T04:18:14.313 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:18:14 smithi089 ceph-mon[64076]: osdmap e106: 8 total, 7 up, 8 in 2024-09-21T04:18:14.314 INFO:journalctl@ceph.osd.3.smithi089.stdout:Sep 21 04:18:14 smithi089 podman[76046]: 2024-09-21 04:18:14.153227457 +0000 UTC m=+0.021070757 image pull 48ed0a2cf89d6d0371a2c6185a5869a747265f6abd5ac2a74f00cc32d5628721 quay.ceph.io/ceph-ci/ceph@sha256:2195e5f178be0a9390728e2a14088236d8fe6f82635f5c84418fe37a04457e00 2024-09-21T04:18:14.314 INFO:journalctl@ceph.osd.3.smithi089.stdout:Sep 21 04:18:14 smithi089 podman[76046]: 2024-09-21 04:18:14.283191062 +0000 UTC m=+0.151034337 container create 464f8f9e062f917572dc2378cef3486f622968311dc3ce597d029790c39b1559 (image=quay.ceph.io/ceph-ci/ceph@sha256:2195e5f178be0a9390728e2a14088236d8fe6f82635f5c84418fe37a04457e00, name=ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-3-deactivate, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, RELEASE=wip-guits-main-2024-09-20-2310-f36283e, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux ) 2024-09-21T04:18:14.370 INFO:teuthology.orchestra.run.smithi089.stdout: 5 210158 9.49 MB/sec execute 413 sec latency 53.113 ms 2024-09-21T04:18:14.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:18:14 smithi195 ceph-mon[49435]: osdmap e106: 8 total, 7 up, 8 in 2024-09-21T04:18:15.352 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:18:15 smithi089 ceph-mon[64076]: pgmap v161: 97 pgs: 6 active+undersized, 18 peering, 2 stale+active+clean, 12 active+undersized+degraded, 59 active+clean; 340 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 3.5 KiB/s rd, 5.3 MiB/s wr, 521 op/s; 101/1674 objects degraded (6.033%) 2024-09-21T04:18:15.352 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:18:15 smithi089 ceph-mon[64076]: Health check failed: Reduced data availability: 2 pgs inactive, 3 pgs peering (PG_AVAILABILITY) 2024-09-21T04:18:15.352 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:18:15 smithi089 ceph-mon[64076]: Health check failed: Degraded data redundancy: 101/1674 objects degraded (6.033%), 12 pgs degraded (PG_DEGRADED) 2024-09-21T04:18:15.352 INFO:journalctl@ceph.osd.3.smithi089.stdout:Sep 21 04:18:15 smithi089 podman[76046]: 2024-09-21 04:18:15.112918524 +0000 UTC m=+0.980761816 container init 464f8f9e062f917572dc2378cef3486f622968311dc3ce597d029790c39b1559 (image=quay.ceph.io/ceph-ci/ceph@sha256:2195e5f178be0a9390728e2a14088236d8fe6f82635f5c84418fe37a04457e00, name=ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-3-deactivate, RELEASE=wip-guits-main-2024-09-20-2310-f36283e, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, GIT_CLEAN=True, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, ceph=True) 2024-09-21T04:18:15.352 INFO:journalctl@ceph.osd.3.smithi089.stdout:Sep 21 04:18:15 smithi089 podman[76046]: 2024-09-21 04:18:15.126754492 +0000 UTC m=+0.994597773 container start 464f8f9e062f917572dc2378cef3486f622968311dc3ce597d029790c39b1559 (image=quay.ceph.io/ceph-ci/ceph@sha256:2195e5f178be0a9390728e2a14088236d8fe6f82635f5c84418fe37a04457e00, name=ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-3-deactivate, GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, ceph=True, io.buildah.version=1.37.2, RELEASE=wip-guits-main-2024-09-20-2310-f36283e, GIT_BRANCH=HEAD, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.license=GPLv2) 2024-09-21T04:18:15.352 INFO:journalctl@ceph.osd.3.smithi089.stdout:Sep 21 04:18:15 smithi089 podman[76046]: 2024-09-21 04:18:15.304132542 +0000 UTC m=+1.171975827 container attach 464f8f9e062f917572dc2378cef3486f622968311dc3ce597d029790c39b1559 (image=quay.ceph.io/ceph-ci/ceph@sha256:2195e5f178be0a9390728e2a14088236d8fe6f82635f5c84418fe37a04457e00, name=ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-3-deactivate, RELEASE=wip-guits-main-2024-09-20-2310-f36283e, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, ceph=True, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS) 2024-09-21T04:18:15.370 INFO:teuthology.orchestra.run.smithi089.stdout: 5 210325 9.48 MB/sec execute 414 sec latency 61.818 ms 2024-09-21T04:18:15.604 INFO:journalctl@ceph.osd.3.smithi089.stdout:Sep 21 04:18:15 smithi089 podman[76046]: 2024-09-21 04:18:15.478845327 +0000 UTC m=+1.346688616 container died 464f8f9e062f917572dc2378cef3486f622968311dc3ce597d029790c39b1559 (image=quay.ceph.io/ceph-ci/ceph@sha256:2195e5f178be0a9390728e2a14088236d8fe6f82635f5c84418fe37a04457e00, name=ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-3-deactivate, RELEASE=wip-guits-main-2024-09-20-2310-f36283e, GIT_CLEAN=True, org.label-schema.schema-version=1.0, ceph=True, CEPH_POINT_RELEASE=, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD) 2024-09-21T04:18:15.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:18:15 smithi195 ceph-mon[49435]: pgmap v161: 97 pgs: 6 active+undersized, 18 peering, 2 stale+active+clean, 12 active+undersized+degraded, 59 active+clean; 340 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 3.5 KiB/s rd, 5.3 MiB/s wr, 521 op/s; 101/1674 objects degraded (6.033%) 2024-09-21T04:18:15.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:18:15 smithi195 ceph-mon[49435]: Health check failed: Reduced data availability: 2 pgs inactive, 3 pgs peering (PG_AVAILABILITY) 2024-09-21T04:18:15.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:18:15 smithi195 ceph-mon[49435]: Health check failed: Degraded data redundancy: 101/1674 objects degraded (6.033%), 12 pgs degraded (PG_DEGRADED) 2024-09-21T04:18:16.370 INFO:teuthology.orchestra.run.smithi089.stdout: 5 210534 9.47 MB/sec execute 415 sec latency 81.918 ms 2024-09-21T04:18:16.641 INFO:journalctl@ceph.osd.3.smithi089.stdout:Sep 21 04:18:16 smithi089 podman[76068]: 2024-09-21 04:18:16.453087983 +0000 UTC m=+0.963668554 container remove 464f8f9e062f917572dc2378cef3486f622968311dc3ce597d029790c39b1559 (image=quay.ceph.io/ceph-ci/ceph@sha256:2195e5f178be0a9390728e2a14088236d8fe6f82635f5c84418fe37a04457e00, name=ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-3-deactivate, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, ceph=True, org.label-schema.schema-version=1.0, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, GIT_CLEAN=True, RELEASE=wip-guits-main-2024-09-20-2310-f36283e, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD) 2024-09-21T04:18:16.641 INFO:journalctl@ceph.osd.3.smithi089.stdout:Sep 21 04:18:16 smithi089 systemd[1]: ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490@osd.3.service: Deactivated successfully. 2024-09-21T04:18:16.641 INFO:journalctl@ceph.osd.3.smithi089.stdout:Sep 21 04:18:16 smithi089 systemd[1]: Stopped Ceph osd.3 for 1b9b79fc-77ce-11ef-baf5-efdc52797490. 2024-09-21T04:18:16.641 INFO:journalctl@ceph.osd.3.smithi089.stdout:Sep 21 04:18:16 smithi089 systemd[1]: ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490@osd.3.service: Consumed 3min 48.252s CPU time. 2024-09-21T04:18:16.893 INFO:journalctl@ceph.osd.3.smithi089.stdout:Sep 21 04:18:16 smithi089 systemd[1]: Starting Ceph osd.3 for 1b9b79fc-77ce-11ef-baf5-efdc52797490... 2024-09-21T04:18:17.145 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:18:17 smithi089 ceph-mon[64076]: pgmap v162: 97 pgs: 7 active+undersized, 18 peering, 16 active+undersized+degraded, 56 active+clean; 336 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.9 KiB/s rd, 4.5 MiB/s wr, 463 op/s; 110/1671 objects degraded (6.583%) 2024-09-21T04:18:17.145 INFO:journalctl@ceph.osd.3.smithi089.stdout:Sep 21 04:18:17 smithi089 podman[76169]: 2024-09-21 04:18:16.97948914 +0000 UTC m=+0.018425697 image pull 48ed0a2cf89d6d0371a2c6185a5869a747265f6abd5ac2a74f00cc32d5628721 quay.ceph.io/ceph-ci/ceph@sha256:2195e5f178be0a9390728e2a14088236d8fe6f82635f5c84418fe37a04457e00 2024-09-21T04:18:17.145 INFO:journalctl@ceph.osd.3.smithi089.stdout:Sep 21 04:18:17 smithi089 podman[76169]: 2024-09-21 04:18:17.098750317 +0000 UTC m=+0.137686864 container create 233d4bd978a2bb5a7738fac438bb3cc7d62cfea7252b9865879f2edf71fc6868 (image=quay.ceph.io/ceph-ci/ceph@sha256:2195e5f178be0a9390728e2a14088236d8fe6f82635f5c84418fe37a04457e00, name=ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-3-activate, org.label-schema.build-date=20240731, ceph=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, io.buildah.version=1.37.2, GIT_CLEAN=True, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, RELEASE=wip-guits-main-2024-09-20-2310-f36283e) 2024-09-21T04:18:17.370 INFO:teuthology.orchestra.run.smithi089.stdout: 5 210835 9.47 MB/sec execute 416 sec latency 51.527 ms 2024-09-21T04:18:17.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:18:17 smithi195 ceph-mon[49435]: pgmap v162: 97 pgs: 7 active+undersized, 18 peering, 16 active+undersized+degraded, 56 active+clean; 336 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.9 KiB/s rd, 4.5 MiB/s wr, 463 op/s; 110/1671 objects degraded (6.583%) 2024-09-21T04:18:17.689 INFO:journalctl@ceph.osd.3.smithi089.stdout:Sep 21 04:18:17 smithi089 podman[76169]: 2024-09-21 04:18:17.410891726 +0000 UTC m=+0.449828284 container init 233d4bd978a2bb5a7738fac438bb3cc7d62cfea7252b9865879f2edf71fc6868 (image=quay.ceph.io/ceph-ci/ceph@sha256:2195e5f178be0a9390728e2a14088236d8fe6f82635f5c84418fe37a04457e00, name=ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-3-activate, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, RELEASE=wip-guits-main-2024-09-20-2310-f36283e, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.build-date=20240731) 2024-09-21T04:18:17.689 INFO:journalctl@ceph.osd.3.smithi089.stdout:Sep 21 04:18:17 smithi089 podman[76169]: 2024-09-21 04:18:17.420663414 +0000 UTC m=+0.459599968 container start 233d4bd978a2bb5a7738fac438bb3cc7d62cfea7252b9865879f2edf71fc6868 (image=quay.ceph.io/ceph-ci/ceph@sha256:2195e5f178be0a9390728e2a14088236d8fe6f82635f5c84418fe37a04457e00, name=ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-3-activate, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, ceph=True, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, RELEASE=wip-guits-main-2024-09-20-2310-f36283e, org.label-schema.license=GPLv2) 2024-09-21T04:18:17.689 INFO:journalctl@ceph.osd.3.smithi089.stdout:Sep 21 04:18:17 smithi089 podman[76169]: 2024-09-21 04:18:17.569922519 +0000 UTC m=+0.608859079 container attach 233d4bd978a2bb5a7738fac438bb3cc7d62cfea7252b9865879f2edf71fc6868 (image=quay.ceph.io/ceph-ci/ceph@sha256:2195e5f178be0a9390728e2a14088236d8fe6f82635f5c84418fe37a04457e00, name=ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-3-activate, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-guits-main-2024-09-20-2310-f36283e, GIT_CLEAN=True, GIT_BRANCH=HEAD, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, ceph=True) 2024-09-21T04:18:17.689 INFO:journalctl@ceph.osd.3.smithi089.stdout:Sep 21 04:18:17 smithi089 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-3-activate[76183]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T04:18:17.689 INFO:journalctl@ceph.osd.3.smithi089.stdout:Sep 21 04:18:17 smithi089 bash[76169]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T04:18:17.689 INFO:journalctl@ceph.osd.3.smithi089.stdout:Sep 21 04:18:17 smithi089 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-3-activate[76183]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T04:18:17.690 INFO:journalctl@ceph.osd.3.smithi089.stdout:Sep 21 04:18:17 smithi089 bash[76169]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T04:18:18.370 INFO:teuthology.orchestra.run.smithi089.stdout: 5 211170 9.46 MB/sec execute 417 sec latency 69.199 ms 2024-09-21T04:18:18.461 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:18:18 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:18:18.461 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:18:18 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T04:18:18.461 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:18:18 smithi089 ceph-mon[64076]: pgmap v163: 97 pgs: 10 active+undersized, 4 peering, 27 active+undersized+degraded, 56 active+clean; 331 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.6 KiB/s rd, 4.4 MiB/s wr, 450 op/s; 197/1620 objects degraded (12.160%) 2024-09-21T04:18:18.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:18:18 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:18:18.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:18:18 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T04:18:18.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:18:18 smithi195 ceph-mon[49435]: pgmap v163: 97 pgs: 10 active+undersized, 4 peering, 27 active+undersized+degraded, 56 active+clean; 331 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.6 KiB/s rd, 4.4 MiB/s wr, 450 op/s; 197/1620 objects degraded (12.160%) 2024-09-21T04:18:19.371 INFO:teuthology.orchestra.run.smithi089.stdout: 5 211507 9.46 MB/sec execute 418 sec latency 55.322 ms 2024-09-21T04:18:19.505 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:18:19 smithi089 ceph-mon[64076]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 3 pgs peering) 2024-09-21T04:18:19.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:18:19 smithi195 ceph-mon[49435]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 3 pgs peering) 2024-09-21T04:18:20.371 INFO:teuthology.orchestra.run.smithi089.stdout: 5 211816 9.46 MB/sec execute 419 sec latency 54.935 ms 2024-09-21T04:18:20.524 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:18:20 smithi089 ceph-mon[64076]: Health check update: Degraded data redundancy: 197/1620 objects degraded (12.160%), 27 pgs degraded (PG_DEGRADED) 2024-09-21T04:18:20.525 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:18:20 smithi089 ceph-mon[64076]: pgmap v164: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 328 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 3.4 KiB/s rd, 5.1 MiB/s wr, 487 op/s; 189/1554 objects degraded (12.162%) 2024-09-21T04:18:20.859 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:18:20 smithi195 ceph-mon[49435]: Health check update: Degraded data redundancy: 197/1620 objects degraded (12.160%), 27 pgs degraded (PG_DEGRADED) 2024-09-21T04:18:20.859 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:18:20 smithi195 ceph-mon[49435]: pgmap v164: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 328 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 3.4 KiB/s rd, 5.1 MiB/s wr, 487 op/s; 189/1554 objects degraded (12.162%) 2024-09-21T04:18:21.371 INFO:teuthology.orchestra.run.smithi089.stdout: 5 212306 9.46 MB/sec execute 420 sec latency 56.818 ms 2024-09-21T04:18:22.307 INFO:journalctl@ceph.osd.3.smithi089.stdout:Sep 21 04:18:22 smithi089 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-3-activate[76183]: --> Failed to activate via raw: did not find any matching OSD to activate 2024-09-21T04:18:22.307 INFO:journalctl@ceph.osd.3.smithi089.stdout:Sep 21 04:18:22 smithi089 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-3-activate[76183]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T04:18:22.307 INFO:journalctl@ceph.osd.3.smithi089.stdout:Sep 21 04:18:22 smithi089 bash[76169]: --> Failed to activate via raw: did not find any matching OSD to activate 2024-09-21T04:18:22.307 INFO:journalctl@ceph.osd.3.smithi089.stdout:Sep 21 04:18:22 smithi089 bash[76169]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T04:18:22.307 INFO:journalctl@ceph.osd.3.smithi089.stdout:Sep 21 04:18:22 smithi089 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-3-activate[76183]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T04:18:22.307 INFO:journalctl@ceph.osd.3.smithi089.stdout:Sep 21 04:18:22 smithi089 bash[76169]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T04:18:22.370 INFO:teuthology.orchestra.run.smithi089.stdout: 5 212694 9.48 MB/sec execute 421 sec latency 64.847 ms 2024-09-21T04:18:22.558 INFO:journalctl@ceph.osd.3.smithi089.stdout:Sep 21 04:18:22 smithi089 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-3-activate[76183]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-09-21T04:18:22.558 INFO:journalctl@ceph.osd.3.smithi089.stdout:Sep 21 04:18:22 smithi089 bash[76169]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-09-21T04:18:22.558 INFO:journalctl@ceph.osd.3.smithi089.stdout:Sep 21 04:18:22 smithi089 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-3-activate[76183]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_1 --path /var/lib/ceph/osd/ceph-3 --no-mon-config 2024-09-21T04:18:22.558 INFO:journalctl@ceph.osd.3.smithi089.stdout:Sep 21 04:18:22 smithi089 bash[76169]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_1 --path /var/lib/ceph/osd/ceph-3 --no-mon-config 2024-09-21T04:18:22.810 INFO:journalctl@ceph.osd.3.smithi089.stdout:Sep 21 04:18:22 smithi089 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-3-activate[76183]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_1 /var/lib/ceph/osd/ceph-3/block 2024-09-21T04:18:22.810 INFO:journalctl@ceph.osd.3.smithi089.stdout:Sep 21 04:18:22 smithi089 bash[76169]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_1 /var/lib/ceph/osd/ceph-3/block 2024-09-21T04:18:22.810 INFO:journalctl@ceph.osd.3.smithi089.stdout:Sep 21 04:18:22 smithi089 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-3-activate[76183]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-3/block 2024-09-21T04:18:22.811 INFO:journalctl@ceph.osd.3.smithi089.stdout:Sep 21 04:18:22 smithi089 bash[76169]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-3/block 2024-09-21T04:18:22.811 INFO:journalctl@ceph.osd.3.smithi089.stdout:Sep 21 04:18:22 smithi089 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-3-activate[76183]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-09-21T04:18:22.811 INFO:journalctl@ceph.osd.3.smithi089.stdout:Sep 21 04:18:22 smithi089 bash[76169]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-09-21T04:18:22.811 INFO:journalctl@ceph.osd.3.smithi089.stdout:Sep 21 04:18:22 smithi089 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-3-activate[76183]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-09-21T04:18:22.811 INFO:journalctl@ceph.osd.3.smithi089.stdout:Sep 21 04:18:22 smithi089 bash[76169]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-09-21T04:18:22.811 INFO:journalctl@ceph.osd.3.smithi089.stdout:Sep 21 04:18:22 smithi089 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-3-activate[76183]: --> ceph-volume lvm activate successful for osd ID: 3 2024-09-21T04:18:22.811 INFO:journalctl@ceph.osd.3.smithi089.stdout:Sep 21 04:18:22 smithi089 bash[76169]: --> ceph-volume lvm activate successful for osd ID: 3 2024-09-21T04:18:22.811 INFO:journalctl@ceph.osd.3.smithi089.stdout:Sep 21 04:18:22 smithi089 podman[76169]: 2024-09-21 04:18:22.789566421 +0000 UTC m=+5.828502982 container died 233d4bd978a2bb5a7738fac438bb3cc7d62cfea7252b9865879f2edf71fc6868 (image=quay.ceph.io/ceph-ci/ceph@sha256:2195e5f178be0a9390728e2a14088236d8fe6f82635f5c84418fe37a04457e00, name=ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-3-activate, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, ceph=True, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-guits-main-2024-09-20-2310-f36283e, GIT_BRANCH=HEAD, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.license=GPLv2) 2024-09-21T04:18:23.067 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:18:23 smithi089 ceph-mon[64076]: pgmap v165: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 328 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.4 KiB/s rd, 4.1 MiB/s wr, 396 op/s; 189/1554 objects degraded (12.162%) 2024-09-21T04:18:23.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:18:23 smithi195 ceph-mon[49435]: pgmap v165: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 328 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.4 KiB/s rd, 4.1 MiB/s wr, 396 op/s; 189/1554 objects degraded (12.162%) 2024-09-21T04:18:23.371 INFO:teuthology.orchestra.run.smithi089.stdout: 5 213013 9.48 MB/sec execute 422 sec latency 36.368 ms 2024-09-21T04:18:24.124 INFO:journalctl@ceph.osd.3.smithi089.stdout:Sep 21 04:18:23 smithi089 podman[76390]: 2024-09-21 04:18:23.87271719 +0000 UTC m=+1.073740781 container remove 233d4bd978a2bb5a7738fac438bb3cc7d62cfea7252b9865879f2edf71fc6868 (image=quay.ceph.io/ceph-ci/ceph@sha256:2195e5f178be0a9390728e2a14088236d8fe6f82635f5c84418fe37a04457e00, name=ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-3-activate, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=wip-guits-main-2024-09-20-2310-f36283e, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux ) 2024-09-21T04:18:24.371 INFO:teuthology.orchestra.run.smithi089.stdout: 5 213491 9.49 MB/sec execute 423 sec latency 45.025 ms 2024-09-21T04:18:24.379 INFO:journalctl@ceph.osd.3.smithi089.stdout:Sep 21 04:18:24 smithi089 podman[76448]: 2024-09-21 04:18:24.044611769 +0000 UTC m=+0.027919778 image pull 48ed0a2cf89d6d0371a2c6185a5869a747265f6abd5ac2a74f00cc32d5628721 quay.ceph.io/ceph-ci/ceph@sha256:2195e5f178be0a9390728e2a14088236d8fe6f82635f5c84418fe37a04457e00 2024-09-21T04:18:24.379 INFO:journalctl@ceph.osd.3.smithi089.stdout:Sep 21 04:18:24 smithi089 podman[76448]: 2024-09-21 04:18:24.181901442 +0000 UTC m=+0.165209433 container create d269cd94bd08f232fd3d269a8e6469ecf78ba2c0c91cefff44da5a2f66bdd9c0 (image=quay.ceph.io/ceph-ci/ceph@sha256:2195e5f178be0a9390728e2a14088236d8fe6f82635f5c84418fe37a04457e00, name=ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-3, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.schema-version=1.0, RELEASE=wip-guits-main-2024-09-20-2310-f36283e, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, ceph=True, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2) 2024-09-21T04:18:24.632 INFO:journalctl@ceph.osd.3.smithi089.stdout:Sep 21 04:18:24 smithi089 podman[76448]: 2024-09-21 04:18:24.569663278 +0000 UTC m=+0.552971272 container init d269cd94bd08f232fd3d269a8e6469ecf78ba2c0c91cefff44da5a2f66bdd9c0 (image=quay.ceph.io/ceph-ci/ceph@sha256:2195e5f178be0a9390728e2a14088236d8fe6f82635f5c84418fe37a04457e00, name=ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-3, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, RELEASE=wip-guits-main-2024-09-20-2310-f36283e, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, ceph=True) 2024-09-21T04:18:24.632 INFO:journalctl@ceph.osd.3.smithi089.stdout:Sep 21 04:18:24 smithi089 podman[76448]: 2024-09-21 04:18:24.577867012 +0000 UTC m=+0.561175022 container start d269cd94bd08f232fd3d269a8e6469ecf78ba2c0c91cefff44da5a2f66bdd9c0 (image=quay.ceph.io/ceph-ci/ceph@sha256:2195e5f178be0a9390728e2a14088236d8fe6f82635f5c84418fe37a04457e00, name=ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-3, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_CLEAN=True, CEPH_POINT_RELEASE=, ceph=True, RELEASE=wip-guits-main-2024-09-20-2310-f36283e, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-21T04:18:24.632 INFO:journalctl@ceph.osd.3.smithi089.stdout:Sep 21 04:18:24 smithi089 ceph-osd[76466]: -- 172.21.15.89:0/170217009 <== mon.0 v2:172.21.15.89:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x564fd42f81e0 con 0x564fd3497c00 2024-09-21T04:18:24.885 INFO:journalctl@ceph.osd.3.smithi089.stdout:Sep 21 04:18:24 smithi089 bash[76448]: d269cd94bd08f232fd3d269a8e6469ecf78ba2c0c91cefff44da5a2f66bdd9c0 2024-09-21T04:18:24.885 INFO:journalctl@ceph.osd.3.smithi089.stdout:Sep 21 04:18:24 smithi089 systemd[1]: Started Ceph osd.3 for 1b9b79fc-77ce-11ef-baf5-efdc52797490. 2024-09-21T04:18:25.141 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:18:25 smithi089 ceph-mon[64076]: pgmap v166: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 341 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 5.8 KiB/s rd, 4.9 MiB/s wr, 456 op/s; 204/1719 objects degraded (11.867%) 2024-09-21T04:18:25.141 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:18:25 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:18:25.141 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:18:25 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:18:25.141 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:18:25 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T04:18:25.141 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:18:25 smithi089 ceph-mon[64076]: Health check update: Degraded data redundancy: 204/1719 objects degraded (11.867%), 27 pgs degraded (PG_DEGRADED) 2024-09-21T04:18:25.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:18:25 smithi195 ceph-mon[49435]: pgmap v166: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 341 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 5.8 KiB/s rd, 4.9 MiB/s wr, 456 op/s; 204/1719 objects degraded (11.867%) 2024-09-21T04:18:25.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:18:25 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:18:25.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:18:25 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:18:25.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:18:25 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T04:18:25.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:18:25 smithi195 ceph-mon[49435]: Health check update: Degraded data redundancy: 204/1719 objects degraded (11.867%), 27 pgs degraded (PG_DEGRADED) 2024-09-21T04:18:25.371 INFO:teuthology.orchestra.run.smithi089.stdout: 5 213783 9.48 MB/sec execute 424 sec latency 43.866 ms 2024-09-21T04:18:26.371 INFO:teuthology.orchestra.run.smithi089.stdout: 5 214117 9.47 MB/sec execute 425 sec latency 32.658 ms 2024-09-21T04:18:26.920 INFO:journalctl@ceph.osd.3.smithi089.stdout:Sep 21 04:18:26 smithi089 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-3[76462]: 2024-09-21T04:18:26.895+0000 7f2dd3007640 -1 Falling back to public interface 2024-09-21T04:18:27.171 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:18:27 smithi089 ceph-mon[64076]: pgmap v167: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 341 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 8.0 KiB/s rd, 3.9 MiB/s wr, 392 op/s; 204/1719 objects degraded (11.867%) 2024-09-21T04:18:27.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:18:27 smithi195 ceph-mon[49435]: pgmap v167: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 341 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 8.0 KiB/s rd, 3.9 MiB/s wr, 392 op/s; 204/1719 objects degraded (11.867%) 2024-09-21T04:18:27.371 INFO:teuthology.orchestra.run.smithi089.stdout: 5 214509 9.47 MB/sec execute 426 sec latency 51.971 ms 2024-09-21T04:18:28.371 INFO:teuthology.orchestra.run.smithi089.stdout: 5 214941 9.48 MB/sec execute 427 sec latency 74.933 ms 2024-09-21T04:18:29.251 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:18:29 smithi089 ceph-mon[64076]: pgmap v168: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 339 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 5.1 MiB/s wr, 501 op/s; 211/1671 objects degraded (12.627%) 2024-09-21T04:18:29.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:18:29 smithi195 ceph-mon[49435]: pgmap v168: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 339 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 5.1 MiB/s wr, 501 op/s; 211/1671 objects degraded (12.627%) 2024-09-21T04:18:29.371 INFO:teuthology.orchestra.run.smithi089.stdout: 5 215265 9.47 MB/sec execute 428 sec latency 54.784 ms 2024-09-21T04:18:30.280 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:18:30 smithi089 ceph-mon[64076]: Health check update: Degraded data redundancy: 211/1671 objects degraded (12.627%), 27 pgs degraded (PG_DEGRADED) 2024-09-21T04:18:30.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:18:30 smithi195 ceph-mon[49435]: Health check update: Degraded data redundancy: 211/1671 objects degraded (12.627%), 27 pgs degraded (PG_DEGRADED) 2024-09-21T04:18:30.371 INFO:teuthology.orchestra.run.smithi089.stdout: 5 215671 9.48 MB/sec execute 429 sec latency 38.065 ms 2024-09-21T04:18:31.308 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:18:31 smithi089 ceph-mon[64076]: pgmap v169: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 336 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 5.0 MiB/s wr, 474 op/s; 201/1596 objects degraded (12.594%) 2024-09-21T04:18:31.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:18:31 smithi195 ceph-mon[49435]: pgmap v169: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 336 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 5.0 MiB/s wr, 474 op/s; 201/1596 objects degraded (12.594%) 2024-09-21T04:18:31.371 INFO:teuthology.orchestra.run.smithi089.stdout: 5 216014 9.47 MB/sec execute 430 sec latency 63.541 ms 2024-09-21T04:18:32.371 INFO:teuthology.orchestra.run.smithi089.stdout: 5 216296 9.48 MB/sec execute 431 sec latency 54.345 ms 2024-09-21T04:18:33.102 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:18:33 smithi089 ceph-mon[64076]: pgmap v170: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 332 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 9.5 KiB/s rd, 4.5 MiB/s wr, 449 op/s; 201/1593 objects degraded (12.618%) 2024-09-21T04:18:33.102 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:18:33 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:18:33.102 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:18:33 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T04:18:33.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:18:33 smithi195 ceph-mon[49435]: pgmap v170: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 332 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 9.5 KiB/s rd, 4.5 MiB/s wr, 449 op/s; 201/1593 objects degraded (12.618%) 2024-09-21T04:18:33.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:18:33 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:18:33.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:18:33 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T04:18:33.371 INFO:teuthology.orchestra.run.smithi089.stdout: 5 216662 9.49 MB/sec execute 432 sec latency 48.790 ms 2024-09-21T04:18:34.368 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:18:34 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:18:34.368 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:18:34 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:18:34.368 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:18:34 smithi089 ceph-mon[64076]: pgmap v171: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 339 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 6.5 KiB/s rd, 6.8 MiB/s wr, 656 op/s; 208/1641 objects degraded (12.675%) 2024-09-21T04:18:34.368 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:18:34 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:18:34.368 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:18:34 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:18:34.371 INFO:teuthology.orchestra.run.smithi089.stdout: 5 217039 9.48 MB/sec execute 433 sec latency 44.371 ms 2024-09-21T04:18:34.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:18:34 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:18:34.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:18:34 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:18:34.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:18:34 smithi195 ceph-mon[49435]: pgmap v171: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 339 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 6.5 KiB/s rd, 6.8 MiB/s wr, 656 op/s; 208/1641 objects degraded (12.675%) 2024-09-21T04:18:34.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:18:34 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:18:34.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:18:34 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:18:35.127 INFO:journalctl@ceph.osd.3.smithi089.stdout:Sep 21 04:18:35 smithi089 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-3[76462]: 2024-09-21T04:18:35.114+0000 7f2dd3007640 -1 osd.3 0 read_superblock omap replica is missing. 2024-09-21T04:18:35.371 INFO:teuthology.orchestra.run.smithi089.stdout: 5 217383 9.48 MB/sec execute 434 sec latency 52.381 ms 2024-09-21T04:18:35.380 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:18:35 smithi089 ceph-mon[64076]: Health check update: Degraded data redundancy: 208/1641 objects degraded (12.675%), 27 pgs degraded (PG_DEGRADED) 2024-09-21T04:18:35.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:18:35 smithi195 ceph-mon[49435]: Health check update: Degraded data redundancy: 208/1641 objects degraded (12.675%), 27 pgs degraded (PG_DEGRADED) 2024-09-21T04:18:36.371 INFO:teuthology.orchestra.run.smithi089.stdout: 5 217611 9.47 MB/sec execute 435 sec latency 49.485 ms 2024-09-21T04:18:36.407 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:18:36 smithi089 ceph-mon[64076]: pgmap v172: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 335 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 9.2 KiB/s rd, 4.3 MiB/s wr, 427 op/s; 208/1638 objects degraded (12.698%) 2024-09-21T04:18:36.408 INFO:journalctl@ceph.osd.3.smithi089.stdout:Sep 21 04:18:36 smithi089 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-3[76462]: 2024-09-21T04:18:36.318+0000 7f2dd3007640 -1 osd.3 104 log_to_monitors true 2024-09-21T04:18:36.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:18:36 smithi195 ceph-mon[49435]: pgmap v172: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 335 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 9.2 KiB/s rd, 4.3 MiB/s wr, 427 op/s; 208/1638 objects degraded (12.698%) 2024-09-21T04:18:37.371 INFO:teuthology.orchestra.run.smithi089.stdout: 5 217899 9.47 MB/sec execute 436 sec latency 77.152 ms 2024-09-21T04:18:37.451 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:18:37 smithi089 ceph-mon[64076]: from='osd.3 [v2:172.21.15.89:6826/848015389,v1:172.21.15.89:6827/848015389]' entity='osd.3' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]} : dispatch 2024-09-21T04:18:37.451 INFO:journalctl@ceph.osd.3.smithi089.stdout:Sep 21 04:18:37 smithi089 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-3[76462]: 2024-09-21T04:18:37.366+0000 7f2dcadb1640 -1 osd.3 104 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-21T04:18:37.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:18:37 smithi195 ceph-mon[49435]: from='osd.3 [v2:172.21.15.89:6826/848015389,v1:172.21.15.89:6827/848015389]' entity='osd.3' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]} : dispatch 2024-09-21T04:18:38.371 INFO:teuthology.orchestra.run.smithi089.stdout: 5 218208 9.46 MB/sec execute 437 sec latency 50.603 ms 2024-09-21T04:18:38.496 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:18:38 smithi089 ceph-mon[64076]: from='osd.3 [v2:172.21.15.89:6826/848015389,v1:172.21.15.89:6827/848015389]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-09-21T04:18:38.496 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:18:38 smithi089 ceph-mon[64076]: osdmap e107: 8 total, 7 up, 8 in 2024-09-21T04:18:38.496 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:18:38 smithi089 ceph-mon[64076]: from='osd.3 [v2:172.21.15.89:6826/848015389,v1:172.21.15.89:6827/848015389]' entity='osd.3' cmd={"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi089", "root=default"]} : dispatch 2024-09-21T04:18:38.496 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:18:38 smithi089 ceph-mon[64076]: pgmap v174: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 336 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 7.7 KiB/s rd, 5.3 MiB/s wr, 576 op/s; 205/1605 objects degraded (12.773%) 2024-09-21T04:18:38.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:18:38 smithi195 ceph-mon[49435]: from='osd.3 [v2:172.21.15.89:6826/848015389,v1:172.21.15.89:6827/848015389]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-09-21T04:18:38.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:18:38 smithi195 ceph-mon[49435]: osdmap e107: 8 total, 7 up, 8 in 2024-09-21T04:18:38.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:18:38 smithi195 ceph-mon[49435]: from='osd.3 [v2:172.21.15.89:6826/848015389,v1:172.21.15.89:6827/848015389]' entity='osd.3' cmd={"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi089", "root=default"]} : dispatch 2024-09-21T04:18:38.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:18:38 smithi195 ceph-mon[49435]: pgmap v174: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 336 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 7.7 KiB/s rd, 5.3 MiB/s wr, 576 op/s; 205/1605 objects degraded (12.773%) 2024-09-21T04:18:39.371 INFO:teuthology.orchestra.run.smithi089.stdout: 5 218313 9.45 MB/sec execute 438 sec latency 1036.428 ms 2024-09-21T04:18:39.592 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:18:39 smithi089 ceph-mon[64076]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-21T04:18:39.592 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:18:39 smithi089 ceph-mon[64076]: osd.3 [v2:172.21.15.89:6826/848015389,v1:172.21.15.89:6827/848015389] boot 2024-09-21T04:18:39.592 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:18:39 smithi089 ceph-mon[64076]: osdmap e108: 8 total, 8 up, 8 in 2024-09-21T04:18:39.592 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:18:39 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "osd metadata", "id": 3} : dispatch 2024-09-21T04:18:39.593 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:18:39 smithi089 ceph-mon[64076]: osdmap e109: 8 total, 8 up, 8 in 2024-09-21T04:18:39.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:18:39 smithi195 ceph-mon[49435]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-21T04:18:39.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:18:39 smithi195 ceph-mon[49435]: osd.3 [v2:172.21.15.89:6826/848015389,v1:172.21.15.89:6827/848015389] boot 2024-09-21T04:18:39.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:18:39 smithi195 ceph-mon[49435]: osdmap e108: 8 total, 8 up, 8 in 2024-09-21T04:18:39.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:18:39 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "osd metadata", "id": 3} : dispatch 2024-09-21T04:18:39.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:18:39 smithi195 ceph-mon[49435]: osdmap e109: 8 total, 8 up, 8 in 2024-09-21T04:18:40.371 INFO:teuthology.orchestra.run.smithi089.stdout: 5 218313 9.42 MB/sec execute 439 sec latency 2036.485 ms 2024-09-21T04:18:40.882 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:18:40 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:18:40.882 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:18:40 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:18:40.882 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:18:40 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T04:18:40.882 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:18:40 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T04:18:40.882 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:18:40 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:18:40.882 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:18:40 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:18:40.882 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:18:40 smithi089 ceph-mon[64076]: Health check update: Degraded data redundancy: 205/1605 objects degraded (12.773%), 27 pgs degraded (PG_DEGRADED) 2024-09-21T04:18:40.882 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:18:40 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T04:18:40.883 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:18:40 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "versions"} : dispatch 2024-09-21T04:18:40.883 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:18:40 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "versions"} : dispatch 2024-09-21T04:18:40.883 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:18:40 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "versions"} : dispatch 2024-09-21T04:18:40.883 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:18:40 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16} : dispatch 2024-09-21T04:18:40.883 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:18:40 smithi089 ceph-mon[64076]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-09-21T04:18:40.883 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:18:40 smithi089 ceph-mon[64076]: Upgrade: unsafe to stop osd(s) at this time (12 PGs are or would become offline) 2024-09-21T04:18:40.883 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:18:40 smithi089 ceph-mon[64076]: pgmap v177: 97 pgs: 12 peering, 10 active+undersized, 19 active+undersized+degraded, 56 active+clean; 334 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 5.3 KiB/s rd, 3.4 MiB/s wr, 383 op/s; 138/1590 objects degraded (8.679%) 2024-09-21T04:18:40.883 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:18:40 smithi089 ceph-mon[64076]: osdmap e110: 8 total, 8 up, 8 in 2024-09-21T04:18:41.027 INFO:teuthology.orchestra.run.smithi089.stdout:true 2024-09-21T04:18:41.109 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:18:40 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:18:41.109 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:18:40 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:18:41.109 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:18:40 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T04:18:41.110 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:18:40 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T04:18:41.110 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:18:40 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:18:41.110 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:18:40 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:18:41.110 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:18:40 smithi195 ceph-mon[49435]: Health check update: Degraded data redundancy: 205/1605 objects degraded (12.773%), 27 pgs degraded (PG_DEGRADED) 2024-09-21T04:18:41.110 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:18:40 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T04:18:41.110 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:18:40 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "versions"} : dispatch 2024-09-21T04:18:41.110 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:18:40 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "versions"} : dispatch 2024-09-21T04:18:41.110 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:18:40 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "versions"} : dispatch 2024-09-21T04:18:41.110 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:18:40 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16} : dispatch 2024-09-21T04:18:41.110 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:18:40 smithi195 ceph-mon[49435]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-09-21T04:18:41.110 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:18:40 smithi195 ceph-mon[49435]: Upgrade: unsafe to stop osd(s) at this time (12 PGs are or would become offline) 2024-09-21T04:18:41.110 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:18:40 smithi195 ceph-mon[49435]: pgmap v177: 97 pgs: 12 peering, 10 active+undersized, 19 active+undersized+degraded, 56 active+clean; 334 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 5.3 KiB/s rd, 3.4 MiB/s wr, 383 op/s; 138/1590 objects degraded (8.679%) 2024-09-21T04:18:41.111 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:18:40 smithi195 ceph-mon[49435]: osdmap e110: 8 total, 8 up, 8 in 2024-09-21T04:18:41.372 INFO:teuthology.orchestra.run.smithi089.stdout: 5 218512 9.41 MB/sec execute 440 sec latency 2383.554 ms 2024-09-21T04:18:41.901 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:18:41 smithi089 ceph-mon[64076]: from='client.34336 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T04:18:41.965 INFO:teuthology.orchestra.run.smithi089.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-21T04:18:41.966 INFO:teuthology.orchestra.run.smithi089.stdout:alertmanager.smithi089 smithi089 *:9093,9094 running (6m) 8s ago 15m 17.9M - 0.27.0 11f11916f8cd 65dfcbcafee1 2024-09-21T04:18:41.966 INFO:teuthology.orchestra.run.smithi089.stdout:crash.smithi089 smithi089 running (3m) 8s ago 15m 6891k - 19.3.0-5103-gf36283e7 48ed0a2cf89d 6c415a5e173e 2024-09-21T04:18:41.966 INFO:teuthology.orchestra.run.smithi089.stdout:crash.smithi195 smithi195 running (3m) 3m ago 15m 6891k - 19.3.0-5103-gf36283e7 48ed0a2cf89d 78ce05a4e9ab 2024-09-21T04:18:41.966 INFO:teuthology.orchestra.run.smithi089.stdout:grafana.smithi089 smithi089 *:3000 running (6m) 8s ago 15m 47.2M - dad864ee21e9 ec4d6738bce7 2024-09-21T04:18:41.966 INFO:teuthology.orchestra.run.smithi089.stdout:mds.foofs.smithi089.osufws smithi089 running (9m) 8s ago 9m 38.5M - 17.2.0 e1d6a67b021e 664d41859bea 2024-09-21T04:18:41.966 INFO:teuthology.orchestra.run.smithi089.stdout:mds.foofs.smithi195.hijxre smithi195 running (9m) 3m ago 9m 16.7M - 17.2.0 e1d6a67b021e 7bc0c6542d12 2024-09-21T04:18:41.966 INFO:teuthology.orchestra.run.smithi089.stdout:mgr.smithi089.srlyxf smithi089 *:8443,9283,8765 running (6m) 8s ago 17m 519M - 19.3.0-5103-gf36283e7 48ed0a2cf89d 3cbeaaa88241 2024-09-21T04:18:41.966 INFO:teuthology.orchestra.run.smithi089.stdout:mgr.smithi195.cgotzm smithi195 *:8443,9283,8765 running (5m) 3m ago 15m 454M - 19.3.0-5103-gf36283e7 48ed0a2cf89d d3734574cf66 2024-09-21T04:18:41.966 INFO:teuthology.orchestra.run.smithi089.stdout:mon.smithi089 smithi089 running (4m) 8s ago 17m 53.6M 2048M 19.3.0-5103-gf36283e7 48ed0a2cf89d 7dd34c8a6f21 2024-09-21T04:18:41.966 INFO:teuthology.orchestra.run.smithi089.stdout:mon.smithi195 smithi195 running (4m) 3m ago 15m 36.7M 2048M 19.3.0-5103-gf36283e7 48ed0a2cf89d 4255937444e7 2024-09-21T04:18:41.966 INFO:teuthology.orchestra.run.smithi089.stdout:nfs.foo.0.0.smithi089.dcjmqc smithi089 *:2049 running (9m) 8s ago 9m 290M - 4.0 e1d6a67b021e 98550f94e773 2024-09-21T04:18:41.966 INFO:teuthology.orchestra.run.smithi089.stdout:nfs.foo.1.0.smithi195.vhoayp smithi195 *:2049 running (9m) 3m ago 9m 88.0M - 4.0 e1d6a67b021e c60a4040130a 2024-09-21T04:18:41.966 INFO:teuthology.orchestra.run.smithi089.stdout:node-exporter.smithi089 smithi089 *:9100 running (6m) 8s ago 15m 9.84M - 1.7.0 72c9c2088986 7ef0d1588475 2024-09-21T04:18:41.966 INFO:teuthology.orchestra.run.smithi089.stdout:node-exporter.smithi195 smithi195 *:9100 running (6m) 3m ago 15m 10.7M - 1.7.0 72c9c2088986 61eafd61bc6c 2024-09-21T04:18:41.967 INFO:teuthology.orchestra.run.smithi089.stdout:osd.0 smithi089 running (2m) 8s ago 14m 331M 1935M 19.3.0-5103-gf36283e7 48ed0a2cf89d 3df0adea59fb 2024-09-21T04:18:41.967 INFO:teuthology.orchestra.run.smithi089.stdout:osd.1 smithi089 running (119s) 8s ago 13m 265M 1935M 19.3.0-5103-gf36283e7 48ed0a2cf89d da2e49774e7e 2024-09-21T04:18:41.967 INFO:teuthology.orchestra.run.smithi089.stdout:osd.2 smithi089 running (68s) 8s ago 13m 202M 1935M 19.3.0-5103-gf36283e7 48ed0a2cf89d b9f6438dd5b9 2024-09-21T04:18:41.967 INFO:teuthology.orchestra.run.smithi089.stdout:osd.3 smithi089 running (17s) 8s ago 13m 11.5M 1935M 19.3.0-5103-gf36283e7 48ed0a2cf89d d269cd94bd08 2024-09-21T04:18:41.967 INFO:teuthology.orchestra.run.smithi089.stdout:osd.4 smithi195 running (12m) 3m ago 12m 524M 2704M 17.2.0 e1d6a67b021e f00e18a1f220 2024-09-21T04:18:41.967 INFO:teuthology.orchestra.run.smithi089.stdout:osd.5 smithi195 running (12m) 3m ago 12m 318M 2704M 17.2.0 e1d6a67b021e cecfd6ff1eee 2024-09-21T04:18:41.967 INFO:teuthology.orchestra.run.smithi089.stdout:osd.6 smithi195 running (11m) 3m ago 11m 510M 2704M 17.2.0 e1d6a67b021e 0d7db362cc23 2024-09-21T04:18:41.967 INFO:teuthology.orchestra.run.smithi089.stdout:osd.7 smithi195 running (11m) 3m ago 11m 387M 2704M 17.2.0 e1d6a67b021e 0109b3fea188 2024-09-21T04:18:41.967 INFO:teuthology.orchestra.run.smithi089.stdout:prometheus.smithi089 smithi089 *:9095 running (5m) 8s ago 14m 53.3M - 2.51.0 1d3b7f56885b 8c9f309a6459 2024-09-21T04:18:42.109 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:18:41 smithi195 ceph-mon[49435]: from='client.34336 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T04:18:42.372 INFO:teuthology.orchestra.run.smithi089.stdout: 5 218719 9.40 MB/sec execute 441 sec latency 82.592 ms 2024-09-21T04:18:42.547 INFO:teuthology.orchestra.run.smithi089.stdout:{ 2024-09-21T04:18:42.547 INFO:teuthology.orchestra.run.smithi089.stdout: "mon": { 2024-09-21T04:18:42.547 INFO:teuthology.orchestra.run.smithi089.stdout: "ceph version 19.3.0-5103-gf36283e7 (f36283e7c91cac1c01cb9a7c4ff21bd697dbd4f2) squid (dev)": 2 2024-09-21T04:18:42.547 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2024-09-21T04:18:42.547 INFO:teuthology.orchestra.run.smithi089.stdout: "mgr": { 2024-09-21T04:18:42.547 INFO:teuthology.orchestra.run.smithi089.stdout: "ceph version 19.3.0-5103-gf36283e7 (f36283e7c91cac1c01cb9a7c4ff21bd697dbd4f2) squid (dev)": 2 2024-09-21T04:18:42.547 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2024-09-21T04:18:42.547 INFO:teuthology.orchestra.run.smithi089.stdout: "osd": { 2024-09-21T04:18:42.548 INFO:teuthology.orchestra.run.smithi089.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 4, 2024-09-21T04:18:42.548 INFO:teuthology.orchestra.run.smithi089.stdout: "ceph version 19.3.0-5103-gf36283e7 (f36283e7c91cac1c01cb9a7c4ff21bd697dbd4f2) squid (dev)": 4 2024-09-21T04:18:42.548 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2024-09-21T04:18:42.548 INFO:teuthology.orchestra.run.smithi089.stdout: "mds": { 2024-09-21T04:18:42.548 INFO:teuthology.orchestra.run.smithi089.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-21T04:18:42.548 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2024-09-21T04:18:42.548 INFO:teuthology.orchestra.run.smithi089.stdout: "overall": { 2024-09-21T04:18:42.548 INFO:teuthology.orchestra.run.smithi089.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 6, 2024-09-21T04:18:42.548 INFO:teuthology.orchestra.run.smithi089.stdout: "ceph version 19.3.0-5103-gf36283e7 (f36283e7c91cac1c01cb9a7c4ff21bd697dbd4f2) squid (dev)": 8 2024-09-21T04:18:42.548 INFO:teuthology.orchestra.run.smithi089.stdout: } 2024-09-21T04:18:42.548 INFO:teuthology.orchestra.run.smithi089.stdout:} 2024-09-21T04:18:42.687 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:18:42 smithi089 ceph-mon[64076]: from='client.34340 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T04:18:42.687 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:18:42 smithi089 ceph-mon[64076]: osdmap e111: 8 total, 8 up, 8 in 2024-09-21T04:18:42.687 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:18:42 smithi089 ceph-mon[64076]: from='client.34344 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T04:18:42.687 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:18:42 smithi089 ceph-mon[64076]: pgmap v180: 97 pgs: 1 activating+undersized+degraded+remapped, 12 peering, 9 active+undersized, 16 active+undersized+degraded, 59 active+clean; 334 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 7.0 KiB/s rd, 1.4 MiB/s wr, 76 op/s; 152/1590 objects degraded (9.560%); 38 KiB/s, 0 objects/s recovering 2024-09-21T04:18:42.687 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:18:42 smithi089 ceph-mon[64076]: from='client.? 172.21.15.89:0/1704120905' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T04:18:42.687 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:18:42 smithi089 ceph-mon[64076]: osdmap e112: 8 total, 8 up, 8 in 2024-09-21T04:18:42.995 INFO:teuthology.orchestra.run.smithi089.stdout:{ 2024-09-21T04:18:42.995 INFO:teuthology.orchestra.run.smithi089.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:2195e5f178be0a9390728e2a14088236d8fe6f82635f5c84418fe37a04457e00", 2024-09-21T04:18:42.995 INFO:teuthology.orchestra.run.smithi089.stdout: "in_progress": true, 2024-09-21T04:18:42.995 INFO:teuthology.orchestra.run.smithi089.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-21T04:18:42.995 INFO:teuthology.orchestra.run.smithi089.stdout: "services_complete": [ 2024-09-21T04:18:42.995 INFO:teuthology.orchestra.run.smithi089.stdout: "mgr", 2024-09-21T04:18:42.995 INFO:teuthology.orchestra.run.smithi089.stdout: "crash", 2024-09-21T04:18:42.995 INFO:teuthology.orchestra.run.smithi089.stdout: "mon" 2024-09-21T04:18:42.995 INFO:teuthology.orchestra.run.smithi089.stdout: ], 2024-09-21T04:18:42.995 INFO:teuthology.orchestra.run.smithi089.stdout: "progress": "10/23 daemons upgraded", 2024-09-21T04:18:42.995 INFO:teuthology.orchestra.run.smithi089.stdout: "message": "Currently upgrading osd daemons", 2024-09-21T04:18:42.995 INFO:teuthology.orchestra.run.smithi089.stdout: "is_paused": false 2024-09-21T04:18:42.995 INFO:teuthology.orchestra.run.smithi089.stdout:} 2024-09-21T04:18:43.109 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:18:42 smithi195 ceph-mon[49435]: from='client.34340 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T04:18:43.109 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:18:42 smithi195 ceph-mon[49435]: osdmap e111: 8 total, 8 up, 8 in 2024-09-21T04:18:43.109 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:18:42 smithi195 ceph-mon[49435]: from='client.34344 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T04:18:43.109 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:18:42 smithi195 ceph-mon[49435]: pgmap v180: 97 pgs: 1 activating+undersized+degraded+remapped, 12 peering, 9 active+undersized, 16 active+undersized+degraded, 59 active+clean; 334 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 7.0 KiB/s rd, 1.4 MiB/s wr, 76 op/s; 152/1590 objects degraded (9.560%); 38 KiB/s, 0 objects/s recovering 2024-09-21T04:18:43.109 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:18:42 smithi195 ceph-mon[49435]: from='client.? 172.21.15.89:0/1704120905' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T04:18:43.109 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:18:42 smithi195 ceph-mon[49435]: osdmap e112: 8 total, 8 up, 8 in 2024-09-21T04:18:43.372 INFO:teuthology.orchestra.run.smithi089.stdout: 5 218852 9.39 MB/sec execute 442 sec latency 695.267 ms 2024-09-21T04:18:43.518 INFO:teuthology.orchestra.run.smithi089.stdout:HEALTH_WARN Degraded data redundancy: 152/1590 objects degraded (9.560%), 17 pgs degraded 2024-09-21T04:18:43.518 INFO:teuthology.orchestra.run.smithi089.stdout:[WRN] PG_DEGRADED: Degraded data redundancy: 152/1590 objects degraded (9.560%), 17 pgs degraded 2024-09-21T04:18:43.518 INFO:teuthology.orchestra.run.smithi089.stdout: pg 2.12 is active+undersized+degraded, acting [5,7] 2024-09-21T04:18:43.518 INFO:teuthology.orchestra.run.smithi089.stdout: pg 2.14 is active+undersized+degraded, acting [6,5] 2024-09-21T04:18:43.518 INFO:teuthology.orchestra.run.smithi089.stdout: pg 2.19 is active+undersized+degraded, acting [6,0] 2024-09-21T04:18:43.518 INFO:teuthology.orchestra.run.smithi089.stdout: pg 2.1b is active+undersized+degraded, acting [7,6] 2024-09-21T04:18:43.518 INFO:teuthology.orchestra.run.smithi089.stdout: pg 2.1e is active+undersized+degraded, acting [0,5] 2024-09-21T04:18:43.518 INFO:teuthology.orchestra.run.smithi089.stdout: pg 3.2 is active+undersized+degraded, acting [5,6] 2024-09-21T04:18:43.518 INFO:teuthology.orchestra.run.smithi089.stdout: pg 3.5 is active+undersized+degraded, acting [5,2] 2024-09-21T04:18:43.518 INFO:teuthology.orchestra.run.smithi089.stdout: pg 3.7 is active+undersized+degraded, acting [7,0] 2024-09-21T04:18:43.519 INFO:teuthology.orchestra.run.smithi089.stdout: pg 3.8 is active+undersized+degraded, acting [1,7] 2024-09-21T04:18:43.519 INFO:teuthology.orchestra.run.smithi089.stdout: pg 3.b is active+undersized+degraded, acting [0,4] 2024-09-21T04:18:43.519 INFO:teuthology.orchestra.run.smithi089.stdout: pg 3.c is active+undersized+degraded, acting [5,6] 2024-09-21T04:18:43.519 INFO:teuthology.orchestra.run.smithi089.stdout: pg 3.15 is active+undersized+degraded, acting [7,4] 2024-09-21T04:18:43.519 INFO:teuthology.orchestra.run.smithi089.stdout: pg 3.18 is active+undersized+degraded, acting [0,1] 2024-09-21T04:18:43.519 INFO:teuthology.orchestra.run.smithi089.stdout: pg 3.1e is activating+undersized+degraded+remapped, acting [2,6] 2024-09-21T04:18:43.519 INFO:teuthology.orchestra.run.smithi089.stdout: pg 4.0 is active+undersized+degraded, acting [7,0] 2024-09-21T04:18:43.519 INFO:teuthology.orchestra.run.smithi089.stdout: pg 4.14 is active+undersized+degraded, acting [2,7] 2024-09-21T04:18:43.519 INFO:teuthology.orchestra.run.smithi089.stdout: pg 4.19 is active+undersized+degraded, acting [2,0] 2024-09-21T04:18:43.748 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:18:43 smithi089 ceph-mon[64076]: from='client.34352 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T04:18:43.748 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:18:43 smithi089 ceph-mon[64076]: from='client.? 172.21.15.89:0/2423380820' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-21T04:18:43.748 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:18:43 smithi089 ceph-mon[64076]: osdmap e113: 8 total, 8 up, 8 in 2024-09-21T04:18:44.109 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:18:43 smithi195 ceph-mon[49435]: from='client.34352 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T04:18:44.109 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:18:43 smithi195 ceph-mon[49435]: from='client.? 172.21.15.89:0/2423380820' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-21T04:18:44.109 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:18:43 smithi195 ceph-mon[49435]: osdmap e113: 8 total, 8 up, 8 in 2024-09-21T04:18:44.372 INFO:teuthology.orchestra.run.smithi089.stdout: 5 219002 9.37 MB/sec execute 443 sec latency 1006.773 ms 2024-09-21T04:18:44.771 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:18:44 smithi089 ceph-mon[64076]: pgmap v183: 97 pgs: 1 active+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 activating+undersized+degraded+remapped, 4 peering, 1 active+recovering+undersized+degraded+remapped, 1 active+undersized+degraded, 85 active+clean; 330 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 7.0 KiB/s rd, 2.6 MiB/s wr, 278 op/s; 169/1521 objects degraded (11.111%); 267 KiB/s, 42 objects/s recovering 2024-09-21T04:18:44.771 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:18:44 smithi089 ceph-mon[64076]: osdmap e114: 8 total, 8 up, 8 in 2024-09-21T04:18:45.109 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:18:44 smithi195 ceph-mon[49435]: pgmap v183: 97 pgs: 1 active+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 activating+undersized+degraded+remapped, 4 peering, 1 active+recovering+undersized+degraded+remapped, 1 active+undersized+degraded, 85 active+clean; 330 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 7.0 KiB/s rd, 2.6 MiB/s wr, 278 op/s; 169/1521 objects degraded (11.111%); 267 KiB/s, 42 objects/s recovering 2024-09-21T04:18:45.109 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:18:44 smithi195 ceph-mon[49435]: osdmap e114: 8 total, 8 up, 8 in 2024-09-21T04:18:45.372 INFO:teuthology.orchestra.run.smithi089.stdout: 5 219237 9.37 MB/sec execute 444 sec latency 777.422 ms 2024-09-21T04:18:45.781 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:18:45 smithi089 ceph-mon[64076]: Health check update: Degraded data redundancy: 169/1521 objects degraded (11.111%), 7 pgs degraded (PG_DEGRADED) 2024-09-21T04:18:45.781 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:18:45 smithi089 ceph-mon[64076]: osdmap e115: 8 total, 8 up, 8 in 2024-09-21T04:18:46.109 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:18:45 smithi195 ceph-mon[49435]: Health check update: Degraded data redundancy: 169/1521 objects degraded (11.111%), 7 pgs degraded (PG_DEGRADED) 2024-09-21T04:18:46.109 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:18:45 smithi195 ceph-mon[49435]: osdmap e115: 8 total, 8 up, 8 in 2024-09-21T04:18:46.372 INFO:teuthology.orchestra.run.smithi089.stdout: 5 219466 9.36 MB/sec execute 445 sec latency 432.974 ms 2024-09-21T04:18:47.139 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:18:46 smithi089 ceph-mon[64076]: osdmap e116: 8 total, 8 up, 8 in 2024-09-21T04:18:47.139 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:18:46 smithi089 ceph-mon[64076]: pgmap v187: 97 pgs: 1 active+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 activating+undersized+degraded+remapped, 4 peering, 1 active+recovering+undersized+degraded+remapped, 1 active+undersized+degraded, 85 active+clean; 330 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 5.4 KiB/s rd, 3.1 MiB/s wr, 334 op/s; 169/1521 objects degraded (11.111%); 275 KiB/s, 51 objects/s recovering 2024-09-21T04:18:47.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:18:46 smithi195 ceph-mon[49435]: osdmap e116: 8 total, 8 up, 8 in 2024-09-21T04:18:47.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:18:46 smithi195 ceph-mon[49435]: pgmap v187: 97 pgs: 1 active+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 activating+undersized+degraded+remapped, 4 peering, 1 active+recovering+undersized+degraded+remapped, 1 active+undersized+degraded, 85 active+clean; 330 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 5.4 KiB/s rd, 3.1 MiB/s wr, 334 op/s; 169/1521 objects degraded (11.111%); 275 KiB/s, 51 objects/s recovering 2024-09-21T04:18:47.375 INFO:teuthology.orchestra.run.smithi089.stdout: 5 219640 9.35 MB/sec execute 446 sec latency 972.224 ms 2024-09-21T04:18:48.143 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:18:47 smithi089 ceph-mon[64076]: osdmap e117: 8 total, 8 up, 8 in 2024-09-21T04:18:48.144 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:18:47 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:18:48.144 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:18:47 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T04:18:48.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:18:47 smithi195 ceph-mon[49435]: osdmap e117: 8 total, 8 up, 8 in 2024-09-21T04:18:48.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:18:47 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:18:48.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:18:47 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T04:18:48.376 INFO:teuthology.orchestra.run.smithi089.stdout: 5 219965 9.36 MB/sec execute 447 sec latency 43.778 ms 2024-09-21T04:18:48.928 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:18:48 smithi089 ceph-mon[64076]: pgmap v189: 97 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 peering, 94 active+clean; 335 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 5.0 KiB/s rd, 3.0 MiB/s wr, 316 op/s; 24/1533 objects degraded (1.566%); 1.2 MiB/s, 41 objects/s recovering 2024-09-21T04:18:49.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:18:48 smithi195 ceph-mon[49435]: pgmap v189: 97 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 peering, 94 active+clean; 335 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 5.0 KiB/s rd, 3.0 MiB/s wr, 316 op/s; 24/1533 objects degraded (1.566%); 1.2 MiB/s, 41 objects/s recovering 2024-09-21T04:18:49.376 INFO:teuthology.orchestra.run.smithi089.stdout: 5 220315 9.37 MB/sec execute 448 sec latency 39.569 ms 2024-09-21T04:18:49.956 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:18:49 smithi089 ceph-mon[64076]: Health check update: Degraded data redundancy: 24/1533 objects degraded (1.566%), 1 pg degraded (PG_DEGRADED) 2024-09-21T04:18:50.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:18:49 smithi195 ceph-mon[49435]: Health check update: Degraded data redundancy: 24/1533 objects degraded (1.566%), 1 pg degraded (PG_DEGRADED) 2024-09-21T04:18:50.376 INFO:teuthology.orchestra.run.smithi089.stdout: 5 220589 9.36 MB/sec execute 449 sec latency 85.440 ms 2024-09-21T04:18:50.988 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:18:50 smithi089 ceph-mon[64076]: pgmap v190: 97 pgs: 97 active+clean; 333 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 3.7 KiB/s rd, 3.4 MiB/s wr, 286 op/s; 996 KiB/s, 35 objects/s recovering 2024-09-21T04:18:51.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:18:50 smithi195 ceph-mon[49435]: pgmap v190: 97 pgs: 97 active+clean; 333 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 3.7 KiB/s rd, 3.4 MiB/s wr, 286 op/s; 996 KiB/s, 35 objects/s recovering 2024-09-21T04:18:51.376 INFO:teuthology.orchestra.run.smithi089.stdout: 5 220791 9.35 MB/sec execute 450 sec latency 139.859 ms 2024-09-21T04:18:52.012 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:18:51 smithi089 ceph-mon[64076]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 24/1533 objects degraded (1.566%), 1 pg degraded) 2024-09-21T04:18:52.012 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:18:51 smithi089 ceph-mon[64076]: Cluster is now healthy 2024-09-21T04:18:52.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:18:51 smithi195 ceph-mon[49435]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 24/1533 objects degraded (1.566%), 1 pg degraded) 2024-09-21T04:18:52.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:18:51 smithi195 ceph-mon[49435]: Cluster is now healthy 2024-09-21T04:18:52.376 INFO:teuthology.orchestra.run.smithi089.stdout: 5 220927 9.34 MB/sec execute 451 sec latency 73.369 ms 2024-09-21T04:18:53.034 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:18:52 smithi089 ceph-mon[64076]: pgmap v191: 97 pgs: 97 active+clean; 333 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 5.9 KiB/s rd, 2.5 MiB/s wr, 218 op/s; 748 KiB/s, 27 objects/s recovering 2024-09-21T04:18:53.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:18:52 smithi195 ceph-mon[49435]: pgmap v191: 97 pgs: 97 active+clean; 333 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 5.9 KiB/s rd, 2.5 MiB/s wr, 218 op/s; 748 KiB/s, 27 objects/s recovering 2024-09-21T04:18:53.376 INFO:teuthology.orchestra.run.smithi089.stdout: 5 221147 9.33 MB/sec execute 452 sec latency 55.889 ms 2024-09-21T04:18:54.377 INFO:teuthology.orchestra.run.smithi089.stdout: 5 221335 9.31 MB/sec execute 453 sec latency 60.569 ms 2024-09-21T04:18:55.135 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:18:55 smithi089 ceph-mon[64076]: pgmap v192: 97 pgs: 97 active+clean; 337 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 3.0 KiB/s rd, 4.4 MiB/s wr, 410 op/s; 656 KiB/s, 23 objects/s recovering 2024-09-21T04:18:55.135 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:18:55 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16} : dispatch 2024-09-21T04:18:55.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:18:55 smithi195 ceph-mon[49435]: pgmap v192: 97 pgs: 97 active+clean; 337 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 3.0 KiB/s rd, 4.4 MiB/s wr, 410 op/s; 656 KiB/s, 23 objects/s recovering 2024-09-21T04:18:55.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:18:55 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16} : dispatch 2024-09-21T04:18:55.376 INFO:teuthology.orchestra.run.smithi089.stdout: 5 221581 9.31 MB/sec execute 454 sec latency 72.102 ms 2024-09-21T04:18:56.159 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:18:56 smithi089 ceph-mon[64076]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-09-21T04:18:56.159 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:18:56 smithi089 ceph-mon[64076]: Upgrade: osd.4 is safe to restart 2024-09-21T04:18:56.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:18:56 smithi195 ceph-mon[49435]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-09-21T04:18:56.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:18:56 smithi195 ceph-mon[49435]: Upgrade: osd.4 is safe to restart 2024-09-21T04:18:56.377 INFO:teuthology.orchestra.run.smithi089.stdout: 5 221866 9.30 MB/sec execute 455 sec latency 89.288 ms 2024-09-21T04:18:57.229 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:18:57 smithi089 ceph-mon[64076]: pgmap v193: 97 pgs: 97 active+clean; 337 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 5.8 KiB/s rd, 3.6 MiB/s wr, 334 op/s; 531 KiB/s, 19 objects/s recovering 2024-09-21T04:18:57.229 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:18:57 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:18:57.229 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:18:57 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "auth get", "entity": "osd.4"} : dispatch 2024-09-21T04:18:57.229 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:18:57 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T04:18:57.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:18:57 smithi195 ceph-mon[49435]: pgmap v193: 97 pgs: 97 active+clean; 337 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 5.8 KiB/s rd, 3.6 MiB/s wr, 334 op/s; 531 KiB/s, 19 objects/s recovering 2024-09-21T04:18:57.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:18:57 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:18:57.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:18:57 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "auth get", "entity": "osd.4"} : dispatch 2024-09-21T04:18:57.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:18:57 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T04:18:57.376 INFO:teuthology.orchestra.run.smithi089.stdout: 5 222215 9.30 MB/sec execute 456 sec latency 51.598 ms 2024-09-21T04:18:58.244 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:18:58 smithi089 ceph-mon[64076]: Upgrade: Updating osd.4 2024-09-21T04:18:58.244 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:18:58 smithi089 ceph-mon[64076]: Deploying daemon osd.4 on smithi195 2024-09-21T04:18:58.244 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:18:58 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:18:58.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:18:58 smithi195 ceph-mon[49435]: Upgrade: Updating osd.4 2024-09-21T04:18:58.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:18:58 smithi195 ceph-mon[49435]: Deploying daemon osd.4 on smithi195 2024-09-21T04:18:58.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:18:58 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:18:58.376 INFO:teuthology.orchestra.run.smithi089.stdout: 5 222560 9.30 MB/sec execute 457 sec latency 47.322 ms 2024-09-21T04:18:59.260 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:18:59 smithi089 ceph-mon[64076]: pgmap v194: 97 pgs: 97 active+clean; 335 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 5.4 KiB/s rd, 4.2 MiB/s wr, 399 op/s; 479 KiB/s, 17 objects/s recovering 2024-09-21T04:18:59.360 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:18:59 smithi195 ceph-mon[49435]: pgmap v194: 97 pgs: 97 active+clean; 335 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 5.4 KiB/s rd, 4.2 MiB/s wr, 399 op/s; 479 KiB/s, 17 objects/s recovering 2024-09-21T04:18:59.376 INFO:teuthology.orchestra.run.smithi089.stdout: 5 222903 9.30 MB/sec execute 458 sec latency 38.951 ms 2024-09-21T04:19:00.377 INFO:teuthology.orchestra.run.smithi089.stdout: 5 223255 9.30 MB/sec execute 459 sec latency 79.062 ms 2024-09-21T04:19:00.846 INFO:journalctl@ceph.osd.4.smithi195.stdout:Sep 21 04:19:00 smithi195 systemd[1]: Stopping Ceph osd.4 for 1b9b79fc-77ce-11ef-baf5-efdc52797490... 2024-09-21T04:19:01.085 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:19:01 smithi089 ceph-mon[64076]: pgmap v195: 97 pgs: 97 active+clean; 336 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 4.8 KiB/s rd, 3.6 MiB/s wr, 361 op/s; 22 KiB/s, 1 objects/s recovering 2024-09-21T04:19:01.085 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:19:01 smithi089 ceph-mon[64076]: osd.4 marked itself down and dead 2024-09-21T04:19:01.109 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:19:01 smithi195 ceph-mon[49435]: pgmap v195: 97 pgs: 97 active+clean; 336 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 4.8 KiB/s rd, 3.6 MiB/s wr, 361 op/s; 22 KiB/s, 1 objects/s recovering 2024-09-21T04:19:01.109 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:19:01 smithi195 ceph-mon[49435]: osd.4 marked itself down and dead 2024-09-21T04:19:01.109 INFO:journalctl@ceph.osd.4.smithi195.stdout:Sep 21 04:19:00 smithi195 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-4[30043]: 2024-09-21T04:19:00.842+0000 7fa19f432700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.4 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-21T04:19:01.109 INFO:journalctl@ceph.osd.4.smithi195.stdout:Sep 21 04:19:00 smithi195 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-4[30043]: 2024-09-21T04:19:00.842+0000 7fa19f432700 -1 osd.4 117 *** Got signal Terminated *** 2024-09-21T04:19:01.109 INFO:journalctl@ceph.osd.4.smithi195.stdout:Sep 21 04:19:00 smithi195 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-4[30043]: 2024-09-21T04:19:00.842+0000 7fa19f432700 -1 osd.4 117 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-21T04:19:01.377 INFO:teuthology.orchestra.run.smithi089.stdout: 5 223517 9.30 MB/sec execute 460 sec latency 307.566 ms 2024-09-21T04:19:01.962 INFO:journalctl@ceph.osd.4.smithi195.stdout:Sep 21 04:19:01 smithi195 podman[52855]: 2024-09-21 04:19:01.67797183 +0000 UTC m=+1.078015081 container died f00e18a1f22083ffc2b666cd49a7417b66cb14aaee83ede784a2060532eed347 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-4, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.k8s.display-name=CentOS Stream 8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., CEPH_POINT_RELEASE=-17.2.0, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vendor=Red Hat, Inc., release=754, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_BRANCH=HEAD, io.buildah.version=1.19.8, distribution-scope=public, version=8, architecture=x86_64, io.openshift.expose-services=, com.redhat.component=centos-stream-container, build-date=2022-05-03T08:36:31.336870, RELEASE=HEAD, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.tags=base centos centos-stream, maintainer=Guillaume Abrioux , GIT_REPO=https://github.com/ceph/ceph-container.git, name=centos-stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, ceph=True, GIT_CLEAN=True, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vcs-type=git) 2024-09-21T04:19:02.346 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:19:02 smithi089 ceph-mon[64076]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-21T04:19:02.346 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:19:02 smithi089 ceph-mon[64076]: osdmap e118: 8 total, 7 up, 8 in 2024-09-21T04:19:02.347 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:19:02 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.4", "id": [0, 4]} : dispatch 2024-09-21T04:19:02.347 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:19:02 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.7", "id": [7, 4]} : dispatch 2024-09-21T04:19:02.347 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:19:02 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.9", "id": [7, 4]} : dispatch 2024-09-21T04:19:02.347 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:19:02 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.e", "id": [7, 4]} : dispatch 2024-09-21T04:19:02.347 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:19:02 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.1a", "id": [7, 2]} : dispatch 2024-09-21T04:19:02.347 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:19:02 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.1d", "id": [7, 4]} : dispatch 2024-09-21T04:19:02.347 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:19:02 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.1f", "id": [0, 1]} : dispatch 2024-09-21T04:19:02.347 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:19:02 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.1", "id": [3, 2]} : dispatch 2024-09-21T04:19:02.347 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:19:02 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.f", "id": [7, 2]} : dispatch 2024-09-21T04:19:02.347 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:19:02 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1a", "id": [3, 2]} : dispatch 2024-09-21T04:19:02.347 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:19:02 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.4", "id": [0, 4]}]': finished 2024-09-21T04:19:02.347 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:19:02 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.7", "id": [7, 4]}]': finished 2024-09-21T04:19:02.347 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:19:02 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.9", "id": [7, 4]}]': finished 2024-09-21T04:19:02.347 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:19:02 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.e", "id": [7, 4]}]': finished 2024-09-21T04:19:02.348 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:19:02 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.1a", "id": [7, 2]}]': finished 2024-09-21T04:19:02.348 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:19:02 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.1d", "id": [7, 4]}]': finished 2024-09-21T04:19:02.348 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:19:02 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.1f", "id": [0, 1]}]': finished 2024-09-21T04:19:02.348 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:19:02 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.1", "id": [3, 2]}]': finished 2024-09-21T04:19:02.348 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:19:02 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.f", "id": [7, 2]}]': finished 2024-09-21T04:19:02.348 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:19:02 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1a", "id": [3, 2]}]': finished 2024-09-21T04:19:02.348 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:19:02 smithi089 ceph-mon[64076]: osdmap e119: 8 total, 7 up, 8 in 2024-09-21T04:19:02.354 INFO:journalctl@ceph.osd.4.smithi195.stdout:Sep 21 04:19:02 smithi195 podman[52855]: 2024-09-21 04:19:02.058564651 +0000 UTC m=+1.458607902 container cleanup f00e18a1f22083ffc2b666cd49a7417b66cb14aaee83ede784a2060532eed347 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-4, name=centos-stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, RELEASE=HEAD, vendor=Red Hat, Inc., description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_CLEAN=True, CEPH_POINT_RELEASE=-17.2.0, maintainer=Guillaume Abrioux , GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.component=centos-stream-container, architecture=x86_64, GIT_BRANCH=HEAD, io.buildah.version=1.19.8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, ceph=True, release=754, io.openshift.tags=base centos centos-stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-type=git, build-date=2022-05-03T08:36:31.336870, io.k8s.display-name=CentOS Stream 8, io.openshift.expose-services=, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, distribution-scope=public, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, version=8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/) 2024-09-21T04:19:02.354 INFO:journalctl@ceph.osd.4.smithi195.stdout:Sep 21 04:19:02 smithi195 bash[52855]: ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-4 2024-09-21T04:19:02.354 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:19:02 smithi195 ceph-mon[49435]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-21T04:19:02.355 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:19:02 smithi195 ceph-mon[49435]: osdmap e118: 8 total, 7 up, 8 in 2024-09-21T04:19:02.355 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:19:02 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.4", "id": [0, 4]} : dispatch 2024-09-21T04:19:02.355 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:19:02 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.7", "id": [7, 4]} : dispatch 2024-09-21T04:19:02.355 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:19:02 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.9", "id": [7, 4]} : dispatch 2024-09-21T04:19:02.355 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:19:02 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.e", "id": [7, 4]} : dispatch 2024-09-21T04:19:02.355 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:19:02 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.1a", "id": [7, 2]} : dispatch 2024-09-21T04:19:02.355 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:19:02 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.1d", "id": [7, 4]} : dispatch 2024-09-21T04:19:02.355 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:19:02 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.1f", "id": [0, 1]} : dispatch 2024-09-21T04:19:02.355 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:19:02 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.1", "id": [3, 2]} : dispatch 2024-09-21T04:19:02.355 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:19:02 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.f", "id": [7, 2]} : dispatch 2024-09-21T04:19:02.355 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:19:02 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1a", "id": [3, 2]} : dispatch 2024-09-21T04:19:02.355 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:19:02 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.4", "id": [0, 4]}]': finished 2024-09-21T04:19:02.355 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:19:02 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.7", "id": [7, 4]}]': finished 2024-09-21T04:19:02.356 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:19:02 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.9", "id": [7, 4]}]': finished 2024-09-21T04:19:02.356 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:19:02 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.e", "id": [7, 4]}]': finished 2024-09-21T04:19:02.356 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:19:02 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.1a", "id": [7, 2]}]': finished 2024-09-21T04:19:02.356 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:19:02 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.1d", "id": [7, 4]}]': finished 2024-09-21T04:19:02.356 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:19:02 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.1f", "id": [0, 1]}]': finished 2024-09-21T04:19:02.356 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:19:02 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.1", "id": [3, 2]}]': finished 2024-09-21T04:19:02.356 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:19:02 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.f", "id": [7, 2]}]': finished 2024-09-21T04:19:02.356 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:19:02 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1a", "id": [3, 2]}]': finished 2024-09-21T04:19:02.356 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:19:02 smithi195 ceph-mon[49435]: osdmap e119: 8 total, 7 up, 8 in 2024-09-21T04:19:02.377 INFO:teuthology.orchestra.run.smithi089.stdout: 5 223668 9.30 MB/sec execute 461 sec latency 1051.182 ms 2024-09-21T04:19:02.609 INFO:journalctl@ceph.osd.4.smithi195.stdout:Sep 21 04:19:02 smithi195 podman[52870]: 2024-09-21 04:19:02.352190024 +0000 UTC m=+0.668467379 container remove f00e18a1f22083ffc2b666cd49a7417b66cb14aaee83ede784a2060532eed347 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-4, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_BRANCH=HEAD, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.openshift.expose-services=, distribution-scope=public, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, architecture=x86_64, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.k8s.display-name=CentOS Stream 8, RELEASE=HEAD, ceph=True, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_CLEAN=True, io.buildah.version=1.19.8, GIT_REPO=https://github.com/ceph/ceph-container.git, version=8, io.openshift.tags=base centos centos-stream, vcs-type=git, name=centos-stream, CEPH_POINT_RELEASE=-17.2.0, release=754, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.component=centos-stream-container, maintainer=Guillaume Abrioux , com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vendor=Red Hat, Inc., build-date=2022-05-03T08:36:31.336870) 2024-09-21T04:19:02.991 INFO:journalctl@ceph.osd.4.smithi195.stdout:Sep 21 04:19:02 smithi195 podman[52961]: 2024-09-21 04:19:02.629809484 +0000 UTC m=+0.016508117 image pull 48ed0a2cf89d6d0371a2c6185a5869a747265f6abd5ac2a74f00cc32d5628721 quay.ceph.io/ceph-ci/ceph@sha256:2195e5f178be0a9390728e2a14088236d8fe6f82635f5c84418fe37a04457e00 2024-09-21T04:19:02.991 INFO:journalctl@ceph.osd.4.smithi195.stdout:Sep 21 04:19:02 smithi195 podman[52961]: 2024-09-21 04:19:02.753395707 +0000 UTC m=+0.140094336 container create de64cf9cd9e3e07f678cbad4bf47fba6182393837cf3d6700c221a9bc8d13e14 (image=quay.ceph.io/ceph-ci/ceph@sha256:2195e5f178be0a9390728e2a14088236d8fe6f82635f5c84418fe37a04457e00, name=ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-4-deactivate, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_CLEAN=True, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, RELEASE=wip-guits-main-2024-09-20-2310-f36283e, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2) 2024-09-21T04:19:03.126 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:19:03 smithi089 ceph-mon[64076]: pgmap v197: 97 pgs: 14 stale+active+clean, 83 active+clean; 332 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.5 KiB/s rd, 3.7 MiB/s wr, 408 op/s 2024-09-21T04:19:03.126 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:19:03 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:19:03.126 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:19:03 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T04:19:03.126 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:19:03 smithi089 ceph-mon[64076]: osdmap e120: 8 total, 7 up, 8 in 2024-09-21T04:19:03.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:19:03 smithi195 ceph-mon[49435]: pgmap v197: 97 pgs: 14 stale+active+clean, 83 active+clean; 332 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.5 KiB/s rd, 3.7 MiB/s wr, 408 op/s 2024-09-21T04:19:03.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:19:03 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:19:03.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:19:03 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T04:19:03.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:19:03 smithi195 ceph-mon[49435]: osdmap e120: 8 total, 7 up, 8 in 2024-09-21T04:19:03.359 INFO:journalctl@ceph.osd.4.smithi195.stdout:Sep 21 04:19:02 smithi195 podman[52961]: 2024-09-21 04:19:02.989610232 +0000 UTC m=+0.376308885 container init de64cf9cd9e3e07f678cbad4bf47fba6182393837cf3d6700c221a9bc8d13e14 (image=quay.ceph.io/ceph-ci/ceph@sha256:2195e5f178be0a9390728e2a14088236d8fe6f82635f5c84418fe37a04457e00, name=ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-4-deactivate, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, ceph=True, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-guits-main-2024-09-20-2310-f36283e, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0) 2024-09-21T04:19:03.359 INFO:journalctl@ceph.osd.4.smithi195.stdout:Sep 21 04:19:02 smithi195 podman[52961]: 2024-09-21 04:19:02.99422615 +0000 UTC m=+0.380924785 container start de64cf9cd9e3e07f678cbad4bf47fba6182393837cf3d6700c221a9bc8d13e14 (image=quay.ceph.io/ceph-ci/ceph@sha256:2195e5f178be0a9390728e2a14088236d8fe6f82635f5c84418fe37a04457e00, name=ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-4-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, RELEASE=wip-guits-main-2024-09-20-2310-f36283e, GIT_CLEAN=True, ceph=True, maintainer=Guillaume Abrioux ) 2024-09-21T04:19:03.359 INFO:journalctl@ceph.osd.4.smithi195.stdout:Sep 21 04:19:03 smithi195 podman[52961]: 2024-09-21 04:19:03.098078655 +0000 UTC m=+0.484777301 container attach de64cf9cd9e3e07f678cbad4bf47fba6182393837cf3d6700c221a9bc8d13e14 (image=quay.ceph.io/ceph-ci/ceph@sha256:2195e5f178be0a9390728e2a14088236d8fe6f82635f5c84418fe37a04457e00, name=ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-4-deactivate, ceph=True, org.label-schema.license=GPLv2, RELEASE=wip-guits-main-2024-09-20-2310-f36283e, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD) 2024-09-21T04:19:03.360 INFO:journalctl@ceph.osd.4.smithi195.stdout:Sep 21 04:19:03 smithi195 podman[52961]: 2024-09-21 04:19:03.197988673 +0000 UTC m=+0.584687316 container died de64cf9cd9e3e07f678cbad4bf47fba6182393837cf3d6700c221a9bc8d13e14 (image=quay.ceph.io/ceph-ci/ceph@sha256:2195e5f178be0a9390728e2a14088236d8fe6f82635f5c84418fe37a04457e00, name=ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-4-deactivate, org.label-schema.schema-version=1.0, RELEASE=wip-guits-main-2024-09-20-2310-f36283e, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731) 2024-09-21T04:19:03.377 INFO:teuthology.orchestra.run.smithi089.stdout: 5 223915 9.28 MB/sec execute 462 sec latency 1122.676 ms 2024-09-21T04:19:04.377 INFO:teuthology.orchestra.run.smithi089.stdout: 5 224048 9.27 MB/sec execute 463 sec latency 1958.181 ms 2024-09-21T04:19:04.609 INFO:journalctl@ceph.osd.4.smithi195.stdout:Sep 21 04:19:04 smithi195 podman[52985]: 2024-09-21 04:19:04.345776544 +0000 UTC m=+1.140543851 container remove de64cf9cd9e3e07f678cbad4bf47fba6182393837cf3d6700c221a9bc8d13e14 (image=quay.ceph.io/ceph-ci/ceph@sha256:2195e5f178be0a9390728e2a14088236d8fe6f82635f5c84418fe37a04457e00, name=ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-4-deactivate, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, RELEASE=wip-guits-main-2024-09-20-2310-f36283e, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0) 2024-09-21T04:19:04.609 INFO:journalctl@ceph.osd.4.smithi195.stdout:Sep 21 04:19:04 smithi195 systemd[1]: ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490@osd.4.service: Deactivated successfully. 2024-09-21T04:19:04.609 INFO:journalctl@ceph.osd.4.smithi195.stdout:Sep 21 04:19:04 smithi195 systemd[1]: Stopped Ceph osd.4 for 1b9b79fc-77ce-11ef-baf5-efdc52797490. 2024-09-21T04:19:04.609 INFO:journalctl@ceph.osd.4.smithi195.stdout:Sep 21 04:19:04 smithi195 systemd[1]: ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490@osd.4.service: Consumed 3min 51.946s CPU time. 2024-09-21T04:19:04.610 INFO:journalctl@ceph.osd.4.smithi195.stdout:Sep 21 04:19:04 smithi195 systemd[1]: Starting Ceph osd.4 for 1b9b79fc-77ce-11ef-baf5-efdc52797490... 2024-09-21T04:19:05.077 INFO:journalctl@ceph.osd.4.smithi195.stdout:Sep 21 04:19:04 smithi195 podman[53082]: 2024-09-21 04:19:04.712127631 +0000 UTC m=+0.019233738 image pull 48ed0a2cf89d6d0371a2c6185a5869a747265f6abd5ac2a74f00cc32d5628721 quay.ceph.io/ceph-ci/ceph@sha256:2195e5f178be0a9390728e2a14088236d8fe6f82635f5c84418fe37a04457e00 2024-09-21T04:19:05.077 INFO:journalctl@ceph.osd.4.smithi195.stdout:Sep 21 04:19:04 smithi195 podman[53082]: 2024-09-21 04:19:04.830703829 +0000 UTC m=+0.137809922 container create 588eac3a2626518405f8420dca51dc48eda3f69353243d4a10b7e8157943de3d (image=quay.ceph.io/ceph-ci/ceph@sha256:2195e5f178be0a9390728e2a14088236d8fe6f82635f5c84418fe37a04457e00, name=ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-4-activate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=wip-guits-main-2024-09-20-2310-f36283e, ceph=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, GIT_CLEAN=True) 2024-09-21T04:19:05.078 INFO:journalctl@ceph.osd.4.smithi195.stdout:Sep 21 04:19:05 smithi195 podman[53082]: 2024-09-21 04:19:05.075430375 +0000 UTC m=+0.382536477 container init 588eac3a2626518405f8420dca51dc48eda3f69353243d4a10b7e8157943de3d (image=quay.ceph.io/ceph-ci/ceph@sha256:2195e5f178be0a9390728e2a14088236d8fe6f82635f5c84418fe37a04457e00, name=ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-4-activate, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-guits-main-2024-09-20-2310-f36283e, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, GIT_CLEAN=True) 2024-09-21T04:19:05.237 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:19:05 smithi089 ceph-mon[64076]: pgmap v200: 97 pgs: 5 remapped+peering, 11 active+undersized, 3 peering, 3 stale+active+clean, 22 active+undersized+degraded, 53 active+clean; 340 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 4.8 KiB/s rd, 4.6 MiB/s wr, 480 op/s; 218/1623 objects degraded (13.432%) 2024-09-21T04:19:05.237 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:19:05 smithi089 ceph-mon[64076]: 2.1a starting backfill to osd.2 from (0'0,0'0] MAX to 95'9413 2024-09-21T04:19:05.237 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:19:05 smithi089 ceph-mon[64076]: osdmap e121: 8 total, 7 up, 8 in 2024-09-21T04:19:05.237 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:19:05 smithi089 ceph-mon[64076]: Health check failed: Degraded data redundancy: 218/1623 objects degraded (13.432%), 22 pgs degraded (PG_DEGRADED) 2024-09-21T04:19:05.237 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:19:05 smithi089 ceph-mon[64076]: osdmap e122: 8 total, 7 up, 8 in 2024-09-21T04:19:05.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:19:05 smithi195 ceph-mon[49435]: pgmap v200: 97 pgs: 5 remapped+peering, 11 active+undersized, 3 peering, 3 stale+active+clean, 22 active+undersized+degraded, 53 active+clean; 340 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 4.8 KiB/s rd, 4.6 MiB/s wr, 480 op/s; 218/1623 objects degraded (13.432%) 2024-09-21T04:19:05.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:19:05 smithi195 ceph-mon[49435]: 2.1a starting backfill to osd.2 from (0'0,0'0] MAX to 95'9413 2024-09-21T04:19:05.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:19:05 smithi195 ceph-mon[49435]: osdmap e121: 8 total, 7 up, 8 in 2024-09-21T04:19:05.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:19:05 smithi195 ceph-mon[49435]: Health check failed: Degraded data redundancy: 218/1623 objects degraded (13.432%), 22 pgs degraded (PG_DEGRADED) 2024-09-21T04:19:05.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:19:05 smithi195 ceph-mon[49435]: osdmap e122: 8 total, 7 up, 8 in 2024-09-21T04:19:05.359 INFO:journalctl@ceph.osd.4.smithi195.stdout:Sep 21 04:19:05 smithi195 podman[53082]: 2024-09-21 04:19:05.078954405 +0000 UTC m=+0.386060498 container start 588eac3a2626518405f8420dca51dc48eda3f69353243d4a10b7e8157943de3d (image=quay.ceph.io/ceph-ci/ceph@sha256:2195e5f178be0a9390728e2a14088236d8fe6f82635f5c84418fe37a04457e00, name=ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-4-activate, GIT_BRANCH=HEAD, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, ceph=True, RELEASE=wip-guits-main-2024-09-20-2310-f36283e, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0) 2024-09-21T04:19:05.359 INFO:journalctl@ceph.osd.4.smithi195.stdout:Sep 21 04:19:05 smithi195 podman[53082]: 2024-09-21 04:19:05.183965597 +0000 UTC m=+0.491071710 container attach 588eac3a2626518405f8420dca51dc48eda3f69353243d4a10b7e8157943de3d (image=quay.ceph.io/ceph-ci/ceph@sha256:2195e5f178be0a9390728e2a14088236d8fe6f82635f5c84418fe37a04457e00, name=ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-4-activate, org.label-schema.build-date=20240731, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, CEPH_POINT_RELEASE=, RELEASE=wip-guits-main-2024-09-20-2310-f36283e, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, ceph=True, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-21T04:19:05.360 INFO:journalctl@ceph.osd.4.smithi195.stdout:Sep 21 04:19:05 smithi195 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-4-activate[53097]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T04:19:05.360 INFO:journalctl@ceph.osd.4.smithi195.stdout:Sep 21 04:19:05 smithi195 bash[53082]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T04:19:05.360 INFO:journalctl@ceph.osd.4.smithi195.stdout:Sep 21 04:19:05 smithi195 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-4-activate[53097]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T04:19:05.360 INFO:journalctl@ceph.osd.4.smithi195.stdout:Sep 21 04:19:05 smithi195 bash[53082]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T04:19:05.377 INFO:teuthology.orchestra.run.smithi089.stdout: 5 224279 9.27 MB/sec execute 464 sec latency 55.405 ms 2024-09-21T04:19:06.377 INFO:teuthology.orchestra.run.smithi089.stdout: 5 224553 9.27 MB/sec execute 465 sec latency 316.427 ms 2024-09-21T04:19:07.031 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:19:06 smithi089 ceph-mon[64076]: osdmap e123: 8 total, 7 up, 8 in 2024-09-21T04:19:07.031 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:19:06 smithi089 ceph-mon[64076]: pgmap v204: 97 pgs: 2 active+clean+remapped, 5 remapped+peering, 12 active+undersized, 3 peering, 24 active+undersized+degraded, 51 active+clean; 341 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.1 KiB/s rd, 5.0 MiB/s wr, 444 op/s; 237/1626 objects degraded (14.576%); 3/1626 objects misplaced (0.185%) 2024-09-21T04:19:07.162 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:19:06 smithi195 ceph-mon[49435]: osdmap e123: 8 total, 7 up, 8 in 2024-09-21T04:19:07.162 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:19:06 smithi195 ceph-mon[49435]: pgmap v204: 97 pgs: 2 active+clean+remapped, 5 remapped+peering, 12 active+undersized, 3 peering, 24 active+undersized+degraded, 51 active+clean; 341 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.1 KiB/s rd, 5.0 MiB/s wr, 444 op/s; 237/1626 objects degraded (14.576%); 3/1626 objects misplaced (0.185%) 2024-09-21T04:19:07.377 INFO:teuthology.orchestra.run.smithi089.stdout: 5 224844 9.26 MB/sec execute 466 sec latency 874.727 ms 2024-09-21T04:19:08.055 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:19:07 smithi089 ceph-mon[64076]: osdmap e124: 8 total, 7 up, 8 in 2024-09-21T04:19:08.172 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:19:07 smithi195 ceph-mon[49435]: osdmap e124: 8 total, 7 up, 8 in 2024-09-21T04:19:08.173 INFO:journalctl@ceph.osd.4.smithi195.stdout:Sep 21 04:19:08 smithi195 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-4-activate[53097]: --> Failed to activate via raw: did not find any matching OSD to activate 2024-09-21T04:19:08.173 INFO:journalctl@ceph.osd.4.smithi195.stdout:Sep 21 04:19:08 smithi195 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-4-activate[53097]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T04:19:08.173 INFO:journalctl@ceph.osd.4.smithi195.stdout:Sep 21 04:19:08 smithi195 bash[53082]: --> Failed to activate via raw: did not find any matching OSD to activate 2024-09-21T04:19:08.173 INFO:journalctl@ceph.osd.4.smithi195.stdout:Sep 21 04:19:08 smithi195 bash[53082]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T04:19:08.173 INFO:journalctl@ceph.osd.4.smithi195.stdout:Sep 21 04:19:08 smithi195 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-4-activate[53097]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T04:19:08.173 INFO:journalctl@ceph.osd.4.smithi195.stdout:Sep 21 04:19:08 smithi195 bash[53082]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T04:19:08.377 INFO:teuthology.orchestra.run.smithi089.stdout: 5 225127 9.25 MB/sec execute 467 sec latency 398.224 ms 2024-09-21T04:19:08.493 INFO:journalctl@ceph.osd.4.smithi195.stdout:Sep 21 04:19:08 smithi195 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-4-activate[53097]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2024-09-21T04:19:08.493 INFO:journalctl@ceph.osd.4.smithi195.stdout:Sep 21 04:19:08 smithi195 bash[53082]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2024-09-21T04:19:08.493 INFO:journalctl@ceph.osd.4.smithi195.stdout:Sep 21 04:19:08 smithi195 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-4-activate[53097]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_4 --path /var/lib/ceph/osd/ceph-4 --no-mon-config 2024-09-21T04:19:08.493 INFO:journalctl@ceph.osd.4.smithi195.stdout:Sep 21 04:19:08 smithi195 bash[53082]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_4 --path /var/lib/ceph/osd/ceph-4 --no-mon-config 2024-09-21T04:19:08.493 INFO:journalctl@ceph.osd.4.smithi195.stdout:Sep 21 04:19:08 smithi195 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-4-activate[53097]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_4 /var/lib/ceph/osd/ceph-4/block 2024-09-21T04:19:08.493 INFO:journalctl@ceph.osd.4.smithi195.stdout:Sep 21 04:19:08 smithi195 bash[53082]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_4 /var/lib/ceph/osd/ceph-4/block 2024-09-21T04:19:08.859 INFO:journalctl@ceph.osd.4.smithi195.stdout:Sep 21 04:19:08 smithi195 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-4-activate[53097]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-4/block 2024-09-21T04:19:08.859 INFO:journalctl@ceph.osd.4.smithi195.stdout:Sep 21 04:19:08 smithi195 bash[53082]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-4/block 2024-09-21T04:19:08.859 INFO:journalctl@ceph.osd.4.smithi195.stdout:Sep 21 04:19:08 smithi195 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-4-activate[53097]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-09-21T04:19:08.859 INFO:journalctl@ceph.osd.4.smithi195.stdout:Sep 21 04:19:08 smithi195 bash[53082]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-09-21T04:19:08.860 INFO:journalctl@ceph.osd.4.smithi195.stdout:Sep 21 04:19:08 smithi195 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-4-activate[53097]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2024-09-21T04:19:08.860 INFO:journalctl@ceph.osd.4.smithi195.stdout:Sep 21 04:19:08 smithi195 bash[53082]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2024-09-21T04:19:08.860 INFO:journalctl@ceph.osd.4.smithi195.stdout:Sep 21 04:19:08 smithi195 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-4-activate[53097]: --> ceph-volume lvm activate successful for osd ID: 4 2024-09-21T04:19:08.860 INFO:journalctl@ceph.osd.4.smithi195.stdout:Sep 21 04:19:08 smithi195 bash[53082]: --> ceph-volume lvm activate successful for osd ID: 4 2024-09-21T04:19:08.860 INFO:journalctl@ceph.osd.4.smithi195.stdout:Sep 21 04:19:08 smithi195 podman[53082]: 2024-09-21 04:19:08.527561924 +0000 UTC m=+3.834668016 container died 588eac3a2626518405f8420dca51dc48eda3f69353243d4a10b7e8157943de3d (image=quay.ceph.io/ceph-ci/ceph@sha256:2195e5f178be0a9390728e2a14088236d8fe6f82635f5c84418fe37a04457e00, name=ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-4-activate, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, GIT_CLEAN=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, CEPH_POINT_RELEASE=, RELEASE=wip-guits-main-2024-09-20-2310-f36283e, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-21T04:19:09.080 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:19:08 smithi089 ceph-mon[64076]: osdmap e125: 8 total, 7 up, 8 in 2024-09-21T04:19:09.080 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:19:08 smithi089 ceph-mon[64076]: pgmap v207: 97 pgs: 4 active+clean+remapped, 6 remapped+peering, 12 active+undersized, 24 active+undersized+degraded, 51 active+clean; 335 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 2.2 MiB/s wr, 181 op/s; 239/1638 objects degraded (14.591%); 13/1638 objects misplaced (0.794%); 1.0 MiB/s, 0 objects/s recovering 2024-09-21T04:19:09.326 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:19:08 smithi195 ceph-mon[49435]: osdmap e125: 8 total, 7 up, 8 in 2024-09-21T04:19:09.326 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:19:08 smithi195 ceph-mon[49435]: pgmap v207: 97 pgs: 4 active+clean+remapped, 6 remapped+peering, 12 active+undersized, 24 active+undersized+degraded, 51 active+clean; 335 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 2.2 MiB/s wr, 181 op/s; 239/1638 objects degraded (14.591%); 13/1638 objects misplaced (0.794%); 1.0 MiB/s, 0 objects/s recovering 2024-09-21T04:19:09.377 INFO:teuthology.orchestra.run.smithi089.stdout: 5 225378 9.25 MB/sec execute 468 sec latency 993.028 ms 2024-09-21T04:19:09.602 INFO:journalctl@ceph.osd.4.smithi195.stdout:Sep 21 04:19:09 smithi195 podman[53311]: 2024-09-21 04:19:09.323913818 +0000 UTC m=+0.789826473 container remove 588eac3a2626518405f8420dca51dc48eda3f69353243d4a10b7e8157943de3d (image=quay.ceph.io/ceph-ci/ceph@sha256:2195e5f178be0a9390728e2a14088236d8fe6f82635f5c84418fe37a04457e00, name=ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-4-activate, CEPH_POINT_RELEASE=, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, RELEASE=wip-guits-main-2024-09-20-2310-f36283e, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, ceph=True, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS) 2024-09-21T04:19:09.602 INFO:journalctl@ceph.osd.4.smithi195.stdout:Sep 21 04:19:09 smithi195 podman[53365]: 2024-09-21 04:19:09.446874002 +0000 UTC m=+0.017368292 image pull 48ed0a2cf89d6d0371a2c6185a5869a747265f6abd5ac2a74f00cc32d5628721 quay.ceph.io/ceph-ci/ceph@sha256:2195e5f178be0a9390728e2a14088236d8fe6f82635f5c84418fe37a04457e00 2024-09-21T04:19:09.860 INFO:journalctl@ceph.osd.4.smithi195.stdout:Sep 21 04:19:09 smithi195 podman[53365]: 2024-09-21 04:19:09.600049117 +0000 UTC m=+0.170543397 container create e375f90f95c434416b24612ad95401ed874031a7b56dcd8d1dfb9e29dcb09e68 (image=quay.ceph.io/ceph-ci/ceph@sha256:2195e5f178be0a9390728e2a14088236d8fe6f82635f5c84418fe37a04457e00, name=ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-4, RELEASE=wip-guits-main-2024-09-20-2310-f36283e, org.label-schema.license=GPLv2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, ceph=True) 2024-09-21T04:19:09.860 INFO:journalctl@ceph.osd.4.smithi195.stdout:Sep 21 04:19:09 smithi195 podman[53365]: 2024-09-21 04:19:09.845300656 +0000 UTC m=+0.415794939 container init e375f90f95c434416b24612ad95401ed874031a7b56dcd8d1dfb9e29dcb09e68 (image=quay.ceph.io/ceph-ci/ceph@sha256:2195e5f178be0a9390728e2a14088236d8fe6f82635f5c84418fe37a04457e00, name=ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-4, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=wip-guits-main-2024-09-20-2310-f36283e, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.vendor=CentOS) 2024-09-21T04:19:09.860 INFO:journalctl@ceph.osd.4.smithi195.stdout:Sep 21 04:19:09 smithi195 podman[53365]: 2024-09-21 04:19:09.849773044 +0000 UTC m=+0.420267324 container start e375f90f95c434416b24612ad95401ed874031a7b56dcd8d1dfb9e29dcb09e68 (image=quay.ceph.io/ceph-ci/ceph@sha256:2195e5f178be0a9390728e2a14088236d8fe6f82635f5c84418fe37a04457e00, name=ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-4, org.label-schema.schema-version=1.0, ceph=True, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, RELEASE=wip-guits-main-2024-09-20-2310-f36283e, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=) 2024-09-21T04:19:10.124 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:19:09 smithi089 ceph-mon[64076]: osdmap e126: 8 total, 7 up, 8 in 2024-09-21T04:19:10.124 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:19:09 smithi089 ceph-mon[64076]: Health check update: Degraded data redundancy: 239/1638 objects degraded (14.591%), 24 pgs degraded (PG_DEGRADED) 2024-09-21T04:19:10.359 INFO:journalctl@ceph.osd.4.smithi195.stdout:Sep 21 04:19:09 smithi195 ceph-osd[53382]: -- 172.21.15.195:0/1633664152 <== mon.1 v2:172.21.15.195:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x559b2d4261e0 con 0x559b2d3b2000 2024-09-21T04:19:10.359 INFO:journalctl@ceph.osd.4.smithi195.stdout:Sep 21 04:19:09 smithi195 bash[53365]: e375f90f95c434416b24612ad95401ed874031a7b56dcd8d1dfb9e29dcb09e68 2024-09-21T04:19:10.359 INFO:journalctl@ceph.osd.4.smithi195.stdout:Sep 21 04:19:09 smithi195 systemd[1]: Started Ceph osd.4 for 1b9b79fc-77ce-11ef-baf5-efdc52797490. 2024-09-21T04:19:10.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:19:09 smithi195 ceph-mon[49435]: osdmap e126: 8 total, 7 up, 8 in 2024-09-21T04:19:10.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:19:09 smithi195 ceph-mon[49435]: Health check update: Degraded data redundancy: 239/1638 objects degraded (14.591%), 24 pgs degraded (PG_DEGRADED) 2024-09-21T04:19:10.377 INFO:teuthology.orchestra.run.smithi089.stdout: 5 225573 9.24 MB/sec execute 469 sec latency 77.464 ms 2024-09-21T04:19:11.150 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:19:11 smithi089 ceph-mon[64076]: pgmap v209: 97 pgs: 8 active+clean+remapped, 2 remapped+peering, 12 active+undersized, 24 active+undersized+degraded, 51 active+clean; 338 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 499 B/s rd, 4.5 MiB/s wr, 461 op/s; 239/1635 objects degraded (14.618%); 38/1635 objects misplaced (2.324%); 1.2 MiB/s, 8 keys/s, 148 objects/s recovering 2024-09-21T04:19:11.150 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:19:11 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:19:11.150 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:19:11 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:19:11.150 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:19:11 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T04:19:11.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:19:11 smithi195 ceph-mon[49435]: pgmap v209: 97 pgs: 8 active+clean+remapped, 2 remapped+peering, 12 active+undersized, 24 active+undersized+degraded, 51 active+clean; 338 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 499 B/s rd, 4.5 MiB/s wr, 461 op/s; 239/1635 objects degraded (14.618%); 38/1635 objects misplaced (2.324%); 1.2 MiB/s, 8 keys/s, 148 objects/s recovering 2024-09-21T04:19:11.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:19:11 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:19:11.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:19:11 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:19:11.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:19:11 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T04:19:11.377 INFO:teuthology.orchestra.run.smithi089.stdout: 5 225873 9.24 MB/sec execute 470 sec latency 81.081 ms 2024-09-21T04:19:11.783 INFO:journalctl@ceph.osd.4.smithi195.stdout:Sep 21 04:19:11 smithi195 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-4[53378]: 2024-09-21T04:19:11.483+0000 7fda0f91a640 -1 Falling back to public interface 2024-09-21T04:19:12.377 INFO:teuthology.orchestra.run.smithi089.stdout: 5 226140 9.23 MB/sec execute 471 sec latency 69.127 ms 2024-09-21T04:19:13.236 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:19:13 smithi089 ceph-mon[64076]: pgmap v210: 97 pgs: 9 active+clean+remapped, 1 remapped+peering, 12 active+undersized, 24 active+undersized+degraded, 51 active+clean; 338 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.3 KiB/s rd, 3.5 MiB/s wr, 333 op/s; 240/1644 objects degraded (14.599%); 48/1644 objects misplaced (2.920%); 916 KiB/s, 5 keys/s, 197 objects/s recovering 2024-09-21T04:19:13.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:19:13 smithi195 ceph-mon[49435]: pgmap v210: 97 pgs: 9 active+clean+remapped, 1 remapped+peering, 12 active+undersized, 24 active+undersized+degraded, 51 active+clean; 338 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.3 KiB/s rd, 3.5 MiB/s wr, 333 op/s; 240/1644 objects degraded (14.599%); 48/1644 objects misplaced (2.920%); 916 KiB/s, 5 keys/s, 197 objects/s recovering 2024-09-21T04:19:13.377 INFO:teuthology.orchestra.run.smithi089.stdout: 5 226384 9.22 MB/sec execute 472 sec latency 82.333 ms 2024-09-21T04:19:14.030 INFO:teuthology.orchestra.run.smithi089.stdout:true 2024-09-21T04:19:14.377 INFO:teuthology.orchestra.run.smithi089.stdout: 5 226707 9.22 MB/sec execute 473 sec latency 50.363 ms 2024-09-21T04:19:14.782 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:19:14 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:19:14.782 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:19:14 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:19:14.782 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:19:14 smithi089 ceph-mon[64076]: pgmap v211: 97 pgs: 10 active+clean+remapped, 12 active+undersized, 24 active+undersized+degraded, 51 active+clean; 336 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 5.4 KiB/s rd, 5.3 MiB/s wr, 527 op/s; 239/1605 objects degraded (14.891%); 44/1605 objects misplaced (2.741%); 775 KiB/s, 5 keys/s, 167 objects/s recovering 2024-09-21T04:19:14.782 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:19:14 smithi089 ceph-mon[64076]: from='client.34362 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T04:19:14.782 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:19:14 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:19:14.782 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:19:14 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:19:14.951 INFO:teuthology.orchestra.run.smithi089.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-21T04:19:14.951 INFO:teuthology.orchestra.run.smithi089.stdout:alertmanager.smithi089 smithi089 *:9093,9094 running (7m) 41s ago 16m 17.9M - 0.27.0 11f11916f8cd 65dfcbcafee1 2024-09-21T04:19:14.951 INFO:teuthology.orchestra.run.smithi089.stdout:crash.smithi089 smithi089 running (4m) 41s ago 16m 6891k - 19.3.0-5103-gf36283e7 48ed0a2cf89d 6c415a5e173e 2024-09-21T04:19:14.951 INFO:teuthology.orchestra.run.smithi089.stdout:crash.smithi195 smithi195 running (3m) 1s ago 15m 6891k - 19.3.0-5103-gf36283e7 48ed0a2cf89d 78ce05a4e9ab 2024-09-21T04:19:14.951 INFO:teuthology.orchestra.run.smithi089.stdout:grafana.smithi089 smithi089 *:3000 running (7m) 41s ago 16m 47.2M - dad864ee21e9 ec4d6738bce7 2024-09-21T04:19:14.951 INFO:teuthology.orchestra.run.smithi089.stdout:mds.foofs.smithi089.osufws smithi089 running (10m) 41s ago 10m 38.5M - 17.2.0 e1d6a67b021e 664d41859bea 2024-09-21T04:19:14.951 INFO:teuthology.orchestra.run.smithi089.stdout:mds.foofs.smithi195.hijxre smithi195 running (10m) 1s ago 10m 17.5M - 17.2.0 e1d6a67b021e 7bc0c6542d12 2024-09-21T04:19:14.951 INFO:teuthology.orchestra.run.smithi089.stdout:mgr.smithi089.srlyxf smithi089 *:8443,9283,8765 running (6m) 41s ago 17m 519M - 19.3.0-5103-gf36283e7 48ed0a2cf89d 3cbeaaa88241 2024-09-21T04:19:14.951 INFO:teuthology.orchestra.run.smithi089.stdout:mgr.smithi195.cgotzm smithi195 *:8443,9283,8765 running (5m) 1s ago 15m 457M - 19.3.0-5103-gf36283e7 48ed0a2cf89d d3734574cf66 2024-09-21T04:19:14.951 INFO:teuthology.orchestra.run.smithi089.stdout:mon.smithi089 smithi089 running (5m) 41s ago 17m 53.6M 2048M 19.3.0-5103-gf36283e7 48ed0a2cf89d 7dd34c8a6f21 2024-09-21T04:19:14.952 INFO:teuthology.orchestra.run.smithi089.stdout:mon.smithi195 smithi195 running (4m) 1s ago 15m 42.9M 2048M 19.3.0-5103-gf36283e7 48ed0a2cf89d 4255937444e7 2024-09-21T04:19:14.952 INFO:teuthology.orchestra.run.smithi089.stdout:nfs.foo.0.0.smithi089.dcjmqc smithi089 *:2049 running (10m) 41s ago 10m 290M - 4.0 e1d6a67b021e 98550f94e773 2024-09-21T04:19:14.952 INFO:teuthology.orchestra.run.smithi089.stdout:nfs.foo.1.0.smithi195.vhoayp smithi195 *:2049 running (10m) 1s ago 10m 88.5M - 4.0 e1d6a67b021e c60a4040130a 2024-09-21T04:19:14.952 INFO:teuthology.orchestra.run.smithi089.stdout:node-exporter.smithi089 smithi089 *:9100 running (7m) 41s ago 16m 9.84M - 1.7.0 72c9c2088986 7ef0d1588475 2024-09-21T04:19:14.952 INFO:teuthology.orchestra.run.smithi089.stdout:node-exporter.smithi195 smithi195 *:9100 running (6m) 1s ago 15m 10.4M - 1.7.0 72c9c2088986 61eafd61bc6c 2024-09-21T04:19:14.952 INFO:teuthology.orchestra.run.smithi089.stdout:osd.0 smithi089 running (3m) 41s ago 15m 331M 1935M 19.3.0-5103-gf36283e7 48ed0a2cf89d 3df0adea59fb 2024-09-21T04:19:14.952 INFO:teuthology.orchestra.run.smithi089.stdout:osd.1 smithi089 running (2m) 41s ago 14m 265M 1935M 19.3.0-5103-gf36283e7 48ed0a2cf89d da2e49774e7e 2024-09-21T04:19:14.952 INFO:teuthology.orchestra.run.smithi089.stdout:osd.2 smithi089 running (101s) 41s ago 14m 202M 1935M 19.3.0-5103-gf36283e7 48ed0a2cf89d b9f6438dd5b9 2024-09-21T04:19:14.952 INFO:teuthology.orchestra.run.smithi089.stdout:osd.3 smithi089 running (50s) 41s ago 13m 11.5M 1935M 19.3.0-5103-gf36283e7 48ed0a2cf89d d269cd94bd08 2024-09-21T04:19:14.952 INFO:teuthology.orchestra.run.smithi089.stdout:osd.4 smithi195 running (5s) 1s ago 13m 14.4M 2704M 19.3.0-5103-gf36283e7 48ed0a2cf89d e375f90f95c4 2024-09-21T04:19:14.952 INFO:teuthology.orchestra.run.smithi089.stdout:osd.5 smithi195 running (12m) 1s ago 12m 527M 2704M 17.2.0 e1d6a67b021e cecfd6ff1eee 2024-09-21T04:19:14.952 INFO:teuthology.orchestra.run.smithi089.stdout:osd.6 smithi195 running (12m) 1s ago 12m 452M 2704M 17.2.0 e1d6a67b021e 0d7db362cc23 2024-09-21T04:19:14.952 INFO:teuthology.orchestra.run.smithi089.stdout:osd.7 smithi195 running (12m) 1s ago 12m 689M 2704M 17.2.0 e1d6a67b021e 0109b3fea188 2024-09-21T04:19:14.952 INFO:teuthology.orchestra.run.smithi089.stdout:prometheus.smithi089 smithi089 *:9095 running (5m) 41s ago 15m 53.3M - 2.51.0 1d3b7f56885b 8c9f309a6459 2024-09-21T04:19:15.014 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:19:14 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:19:15.014 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:19:14 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:19:15.014 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:19:14 smithi195 ceph-mon[49435]: pgmap v211: 97 pgs: 10 active+clean+remapped, 12 active+undersized, 24 active+undersized+degraded, 51 active+clean; 336 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 5.4 KiB/s rd, 5.3 MiB/s wr, 527 op/s; 239/1605 objects degraded (14.891%); 44/1605 objects misplaced (2.741%); 775 KiB/s, 5 keys/s, 167 objects/s recovering 2024-09-21T04:19:15.014 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:19:14 smithi195 ceph-mon[49435]: from='client.34362 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T04:19:15.014 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:19:14 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:19:15.014 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:19:14 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:19:15.377 INFO:teuthology.orchestra.run.smithi089.stdout: 5 226967 9.22 MB/sec execute 474 sec latency 53.674 ms 2024-09-21T04:19:15.522 INFO:teuthology.orchestra.run.smithi089.stdout:{ 2024-09-21T04:19:15.522 INFO:teuthology.orchestra.run.smithi089.stdout: "mon": { 2024-09-21T04:19:15.522 INFO:teuthology.orchestra.run.smithi089.stdout: "ceph version 19.3.0-5103-gf36283e7 (f36283e7c91cac1c01cb9a7c4ff21bd697dbd4f2) squid (dev)": 2 2024-09-21T04:19:15.522 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2024-09-21T04:19:15.522 INFO:teuthology.orchestra.run.smithi089.stdout: "mgr": { 2024-09-21T04:19:15.522 INFO:teuthology.orchestra.run.smithi089.stdout: "ceph version 19.3.0-5103-gf36283e7 (f36283e7c91cac1c01cb9a7c4ff21bd697dbd4f2) squid (dev)": 2 2024-09-21T04:19:15.522 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2024-09-21T04:19:15.522 INFO:teuthology.orchestra.run.smithi089.stdout: "osd": { 2024-09-21T04:19:15.522 INFO:teuthology.orchestra.run.smithi089.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3, 2024-09-21T04:19:15.522 INFO:teuthology.orchestra.run.smithi089.stdout: "ceph version 19.3.0-5103-gf36283e7 (f36283e7c91cac1c01cb9a7c4ff21bd697dbd4f2) squid (dev)": 4 2024-09-21T04:19:15.522 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2024-09-21T04:19:15.522 INFO:teuthology.orchestra.run.smithi089.stdout: "mds": { 2024-09-21T04:19:15.522 INFO:teuthology.orchestra.run.smithi089.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-21T04:19:15.523 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2024-09-21T04:19:15.523 INFO:teuthology.orchestra.run.smithi089.stdout: "overall": { 2024-09-21T04:19:15.523 INFO:teuthology.orchestra.run.smithi089.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 5, 2024-09-21T04:19:15.523 INFO:teuthology.orchestra.run.smithi089.stdout: "ceph version 19.3.0-5103-gf36283e7 (f36283e7c91cac1c01cb9a7c4ff21bd697dbd4f2) squid (dev)": 8 2024-09-21T04:19:15.523 INFO:teuthology.orchestra.run.smithi089.stdout: } 2024-09-21T04:19:15.523 INFO:teuthology.orchestra.run.smithi089.stdout:} 2024-09-21T04:19:15.803 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:19:15 smithi089 ceph-mon[64076]: from='client.34366 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T04:19:15.803 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:19:15 smithi089 ceph-mon[64076]: Health check update: Degraded data redundancy: 239/1605 objects degraded (14.891%), 24 pgs degraded (PG_DEGRADED) 2024-09-21T04:19:15.803 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:19:15 smithi089 ceph-mon[64076]: from='client.34370 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T04:19:15.803 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:19:15 smithi089 ceph-mon[64076]: from='client.? 172.21.15.89:0/1570142929' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T04:19:15.995 INFO:teuthology.orchestra.run.smithi089.stdout:{ 2024-09-21T04:19:15.995 INFO:teuthology.orchestra.run.smithi089.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:2195e5f178be0a9390728e2a14088236d8fe6f82635f5c84418fe37a04457e00", 2024-09-21T04:19:15.995 INFO:teuthology.orchestra.run.smithi089.stdout: "in_progress": true, 2024-09-21T04:19:15.995 INFO:teuthology.orchestra.run.smithi089.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-21T04:19:15.995 INFO:teuthology.orchestra.run.smithi089.stdout: "services_complete": [ 2024-09-21T04:19:15.995 INFO:teuthology.orchestra.run.smithi089.stdout: "mgr", 2024-09-21T04:19:15.995 INFO:teuthology.orchestra.run.smithi089.stdout: "crash", 2024-09-21T04:19:15.995 INFO:teuthology.orchestra.run.smithi089.stdout: "mon" 2024-09-21T04:19:15.995 INFO:teuthology.orchestra.run.smithi089.stdout: ], 2024-09-21T04:19:15.996 INFO:teuthology.orchestra.run.smithi089.stdout: "progress": "11/23 daemons upgraded", 2024-09-21T04:19:15.996 INFO:teuthology.orchestra.run.smithi089.stdout: "message": "Currently upgrading osd daemons", 2024-09-21T04:19:15.996 INFO:teuthology.orchestra.run.smithi089.stdout: "is_paused": false 2024-09-21T04:19:15.996 INFO:teuthology.orchestra.run.smithi089.stdout:} 2024-09-21T04:19:16.006 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:19:15 smithi195 ceph-mon[49435]: from='client.34366 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T04:19:16.007 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:19:15 smithi195 ceph-mon[49435]: Health check update: Degraded data redundancy: 239/1605 objects degraded (14.891%), 24 pgs degraded (PG_DEGRADED) 2024-09-21T04:19:16.007 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:19:15 smithi195 ceph-mon[49435]: from='client.34370 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T04:19:16.007 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:19:15 smithi195 ceph-mon[49435]: from='client.? 172.21.15.89:0/1570142929' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T04:19:16.377 INFO:teuthology.orchestra.run.smithi089.stdout: 5 227263 9.23 MB/sec execute 475 sec latency 49.127 ms 2024-09-21T04:19:16.535 INFO:teuthology.orchestra.run.smithi089.stdout:HEALTH_WARN 1 osds down; Degraded data redundancy: 239/1605 objects degraded (14.891%), 24 pgs degraded 2024-09-21T04:19:16.535 INFO:teuthology.orchestra.run.smithi089.stdout:[WRN] OSD_DOWN: 1 osds down 2024-09-21T04:19:16.535 INFO:teuthology.orchestra.run.smithi089.stdout: osd.4 (root=default,host=smithi195) is down 2024-09-21T04:19:16.536 INFO:teuthology.orchestra.run.smithi089.stdout:[WRN] PG_DEGRADED: Degraded data redundancy: 239/1605 objects degraded (14.891%), 24 pgs degraded 2024-09-21T04:19:16.536 INFO:teuthology.orchestra.run.smithi089.stdout: pg 2.5 is active+undersized+degraded, acting [7,0] 2024-09-21T04:19:16.536 INFO:teuthology.orchestra.run.smithi089.stdout: pg 2.6 is active+undersized+degraded, acting [1,6] 2024-09-21T04:19:16.536 INFO:teuthology.orchestra.run.smithi089.stdout: pg 2.b is active+undersized+degraded, acting [2,5] 2024-09-21T04:19:16.536 INFO:teuthology.orchestra.run.smithi089.stdout: pg 2.d is active+undersized+degraded, acting [1,3] 2024-09-21T04:19:16.537 INFO:teuthology.orchestra.run.smithi089.stdout: pg 2.f is active+undersized+degraded, acting [0,7] 2024-09-21T04:19:16.537 INFO:teuthology.orchestra.run.smithi089.stdout: pg 2.11 is active+undersized+degraded, acting [6,1] 2024-09-21T04:19:16.537 INFO:teuthology.orchestra.run.smithi089.stdout: pg 2.13 is active+undersized+degraded, acting [0,3] 2024-09-21T04:19:16.537 INFO:teuthology.orchestra.run.smithi089.stdout: pg 2.18 is active+undersized+degraded, acting [5,7] 2024-09-21T04:19:16.537 INFO:teuthology.orchestra.run.smithi089.stdout: pg 3.3 is active+undersized+degraded, acting [0,6] 2024-09-21T04:19:16.537 INFO:teuthology.orchestra.run.smithi089.stdout: pg 3.6 is active+undersized+degraded, acting [0,1] 2024-09-21T04:19:16.537 INFO:teuthology.orchestra.run.smithi089.stdout: pg 3.9 is active+undersized+degraded, acting [2,7] 2024-09-21T04:19:16.538 INFO:teuthology.orchestra.run.smithi089.stdout: pg 3.a is active+undersized+degraded, acting [6,1] 2024-09-21T04:19:16.538 INFO:teuthology.orchestra.run.smithi089.stdout: pg 3.b is active+undersized+degraded, acting [3,0] 2024-09-21T04:19:16.538 INFO:teuthology.orchestra.run.smithi089.stdout: pg 3.e is active+undersized+degraded, acting [7,1] 2024-09-21T04:19:16.538 INFO:teuthology.orchestra.run.smithi089.stdout: pg 3.11 is active+undersized+degraded, acting [7,6] 2024-09-21T04:19:16.538 INFO:teuthology.orchestra.run.smithi089.stdout: pg 3.13 is active+undersized+degraded, acting [7,2] 2024-09-21T04:19:16.538 INFO:teuthology.orchestra.run.smithi089.stdout: pg 3.14 is active+undersized+degraded, acting [7,6] 2024-09-21T04:19:16.539 INFO:teuthology.orchestra.run.smithi089.stdout: pg 3.15 is active+undersized+degraded, acting [7,3] 2024-09-21T04:19:16.539 INFO:teuthology.orchestra.run.smithi089.stdout: pg 3.19 is active+undersized+degraded, acting [1,3] 2024-09-21T04:19:16.539 INFO:teuthology.orchestra.run.smithi089.stdout: pg 3.1a is active+undersized+degraded, acting [1,2] 2024-09-21T04:19:16.539 INFO:teuthology.orchestra.run.smithi089.stdout: pg 3.1b is active+undersized+degraded, acting [0,7] 2024-09-21T04:19:16.539 INFO:teuthology.orchestra.run.smithi089.stdout: pg 3.1c is active+undersized+degraded, acting [5,1] 2024-09-21T04:19:16.539 INFO:teuthology.orchestra.run.smithi089.stdout: pg 3.1d is active+undersized+degraded, acting [5,6] 2024-09-21T04:19:16.540 INFO:teuthology.orchestra.run.smithi089.stdout: pg 4.5 is active+undersized+degraded, acting [6,7] 2024-09-21T04:19:16.839 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:19:16 smithi089 ceph-mon[64076]: from='client.34378 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T04:19:16.839 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:19:16 smithi089 ceph-mon[64076]: pgmap v212: 97 pgs: 10 active+clean+remapped, 12 active+undersized, 24 active+undersized+degraded, 51 active+clean; 335 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.9 KiB/s rd, 4.5 MiB/s wr, 461 op/s; 241/1617 objects degraded (14.904%); 46/1617 objects misplaced (2.845%); 682 KiB/s, 4 keys/s, 147 objects/s recovering 2024-09-21T04:19:16.839 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:19:16 smithi089 ceph-mon[64076]: from='client.? 172.21.15.89:0/4098444917' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-21T04:19:16.860 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:19:16 smithi195 ceph-mon[49435]: from='client.34378 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T04:19:16.860 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:19:16 smithi195 ceph-mon[49435]: pgmap v212: 97 pgs: 10 active+clean+remapped, 12 active+undersized, 24 active+undersized+degraded, 51 active+clean; 335 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.9 KiB/s rd, 4.5 MiB/s wr, 461 op/s; 241/1617 objects degraded (14.904%); 46/1617 objects misplaced (2.845%); 682 KiB/s, 4 keys/s, 147 objects/s recovering 2024-09-21T04:19:16.860 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:19:16 smithi195 ceph-mon[49435]: from='client.? 172.21.15.89:0/4098444917' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-21T04:19:17.359 INFO:journalctl@ceph.osd.4.smithi195.stdout:Sep 21 04:19:17 smithi195 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-4[53378]: 2024-09-21T04:19:17.096+0000 7fda0f91a640 -1 osd.4 0 read_superblock omap replica is missing. 2024-09-21T04:19:17.377 INFO:teuthology.orchestra.run.smithi089.stdout: 5 227537 9.22 MB/sec execute 476 sec latency 78.844 ms 2024-09-21T04:19:18.377 INFO:teuthology.orchestra.run.smithi089.stdout: 5 227841 9.22 MB/sec execute 477 sec latency 73.992 ms 2024-09-21T04:19:18.389 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:19:18 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:19:18.389 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:19:18 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T04:19:18.389 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:19:18 smithi089 ceph-mon[64076]: pgmap v213: 97 pgs: 10 active+clean+remapped, 12 active+undersized, 24 active+undersized+degraded, 51 active+clean; 338 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.6 KiB/s rd, 4.4 MiB/s wr, 463 op/s; 246/1662 objects degraded (14.801%); 46/1662 objects misplaced (2.768%); 140 KiB/s, 3 keys/s, 118 objects/s recovering 2024-09-21T04:19:18.389 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:19:18 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:19:18.389 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:19:18 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:19:18.389 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:19:18 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T04:19:18.389 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:19:18 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T04:19:18.389 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:19:18 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:19:18.389 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:19:18 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:19:18.389 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:19:18 smithi089 ceph-mon[64076]: from='osd.4 [v2:172.21.15.195:6800/3047762682,v1:172.21.15.195:6801/3047762682]' entity='osd.4' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]} : dispatch 2024-09-21T04:19:18.390 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:19:18 smithi089 ceph-mon[64076]: from='osd.4 ' entity='osd.4' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]} : dispatch 2024-09-21T04:19:18.609 INFO:journalctl@ceph.osd.4.smithi195.stdout:Sep 21 04:19:18 smithi195 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-4[53378]: 2024-09-21T04:19:18.330+0000 7fda0f91a640 -1 osd.4 117 log_to_monitors true 2024-09-21T04:19:18.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:19:18 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:19:18.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:19:18 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T04:19:18.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:19:18 smithi195 ceph-mon[49435]: pgmap v213: 97 pgs: 10 active+clean+remapped, 12 active+undersized, 24 active+undersized+degraded, 51 active+clean; 338 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.6 KiB/s rd, 4.4 MiB/s wr, 463 op/s; 246/1662 objects degraded (14.801%); 46/1662 objects misplaced (2.768%); 140 KiB/s, 3 keys/s, 118 objects/s recovering 2024-09-21T04:19:18.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:19:18 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:19:18.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:19:18 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:19:18.610 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:19:18 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T04:19:18.610 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:19:18 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T04:19:18.610 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:19:18 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:19:18.610 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:19:18 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:19:18.610 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:19:18 smithi195 ceph-mon[49435]: from='osd.4 [v2:172.21.15.195:6800/3047762682,v1:172.21.15.195:6801/3047762682]' entity='osd.4' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]} : dispatch 2024-09-21T04:19:18.610 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:19:18 smithi195 ceph-mon[49435]: from='osd.4 ' entity='osd.4' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]} : dispatch 2024-09-21T04:19:19.378 INFO:teuthology.orchestra.run.smithi089.stdout: 5 228063 9.21 MB/sec execute 478 sec latency 54.183 ms 2024-09-21T04:19:19.432 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:19:19 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T04:19:19.432 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:19:19 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "versions"} : dispatch 2024-09-21T04:19:19.432 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:19:19 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "versions"} : dispatch 2024-09-21T04:19:19.432 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:19:19 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "versions"} : dispatch 2024-09-21T04:19:19.432 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:19:19 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16} : dispatch 2024-09-21T04:19:19.432 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:19:19 smithi089 ceph-mon[64076]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-09-21T04:19:19.432 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:19:19 smithi089 ceph-mon[64076]: Upgrade: unsafe to stop osd(s) at this time (7 PGs are or would become offline) 2024-09-21T04:19:19.432 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:19:19 smithi089 ceph-mon[64076]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-09-21T04:19:19.432 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:19:19 smithi089 ceph-mon[64076]: from='osd.4 [v2:172.21.15.195:6800/3047762682,v1:172.21.15.195:6801/3047762682]' entity='osd.4' cmd={"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi195", "root=default"]} : dispatch 2024-09-21T04:19:19.433 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:19:19 smithi089 ceph-mon[64076]: osdmap e127: 8 total, 7 up, 8 in 2024-09-21T04:19:19.433 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:19:19 smithi089 ceph-mon[64076]: from='osd.4 ' entity='osd.4' cmd={"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi195", "root=default"]} : dispatch 2024-09-21T04:19:19.609 INFO:journalctl@ceph.osd.4.smithi195.stdout:Sep 21 04:19:19 smithi195 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-4[53378]: 2024-09-21T04:19:19.127+0000 7fda076c4640 -1 osd.4 117 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-21T04:19:19.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:19:19 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T04:19:19.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:19:19 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "versions"} : dispatch 2024-09-21T04:19:19.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:19:19 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "versions"} : dispatch 2024-09-21T04:19:19.610 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:19:19 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "versions"} : dispatch 2024-09-21T04:19:19.610 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:19:19 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16} : dispatch 2024-09-21T04:19:19.610 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:19:19 smithi195 ceph-mon[49435]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-09-21T04:19:19.610 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:19:19 smithi195 ceph-mon[49435]: Upgrade: unsafe to stop osd(s) at this time (7 PGs are or would become offline) 2024-09-21T04:19:19.610 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:19:19 smithi195 ceph-mon[49435]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-09-21T04:19:19.610 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:19:19 smithi195 ceph-mon[49435]: from='osd.4 [v2:172.21.15.195:6800/3047762682,v1:172.21.15.195:6801/3047762682]' entity='osd.4' cmd={"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi195", "root=default"]} : dispatch 2024-09-21T04:19:19.610 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:19:19 smithi195 ceph-mon[49435]: osdmap e127: 8 total, 7 up, 8 in 2024-09-21T04:19:19.610 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:19:19 smithi195 ceph-mon[49435]: from='osd.4 ' entity='osd.4' cmd={"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi195", "root=default"]} : dispatch 2024-09-21T04:19:20.377 INFO:teuthology.orchestra.run.smithi089.stdout: 5 228341 9.21 MB/sec execute 479 sec latency 259.702 ms 2024-09-21T04:19:20.467 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:19:20 smithi089 ceph-mon[64076]: Health check update: Degraded data redundancy: 246/1662 objects degraded (14.801%), 24 pgs degraded (PG_DEGRADED) 2024-09-21T04:19:20.467 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:19:20 smithi089 ceph-mon[64076]: pgmap v215: 97 pgs: 10 active+clean+remapped, 12 active+undersized, 24 active+undersized+degraded, 51 active+clean; 339 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 6.6 KiB/s rd, 4.0 MiB/s wr, 364 op/s; 242/1641 objects degraded (14.747%); 48/1641 objects misplaced (2.925%); 53 KiB/s, 57 objects/s recovering 2024-09-21T04:19:20.467 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:19:20 smithi089 ceph-mon[64076]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-21T04:19:20.467 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:19:20 smithi089 ceph-mon[64076]: osd.4 [v2:172.21.15.195:6800/3047762682,v1:172.21.15.195:6801/3047762682] boot 2024-09-21T04:19:20.467 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:19:20 smithi089 ceph-mon[64076]: osdmap e128: 8 total, 8 up, 8 in 2024-09-21T04:19:20.467 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:19:20 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "osd metadata", "id": 4} : dispatch 2024-09-21T04:19:20.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:19:20 smithi195 ceph-mon[49435]: Health check update: Degraded data redundancy: 246/1662 objects degraded (14.801%), 24 pgs degraded (PG_DEGRADED) 2024-09-21T04:19:20.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:19:20 smithi195 ceph-mon[49435]: pgmap v215: 97 pgs: 10 active+clean+remapped, 12 active+undersized, 24 active+undersized+degraded, 51 active+clean; 339 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 6.6 KiB/s rd, 4.0 MiB/s wr, 364 op/s; 242/1641 objects degraded (14.747%); 48/1641 objects misplaced (2.925%); 53 KiB/s, 57 objects/s recovering 2024-09-21T04:19:20.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:19:20 smithi195 ceph-mon[49435]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-21T04:19:20.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:19:20 smithi195 ceph-mon[49435]: osd.4 [v2:172.21.15.195:6800/3047762682,v1:172.21.15.195:6801/3047762682] boot 2024-09-21T04:19:20.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:19:20 smithi195 ceph-mon[49435]: osdmap e128: 8 total, 8 up, 8 in 2024-09-21T04:19:20.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:19:20 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "osd metadata", "id": 4} : dispatch 2024-09-21T04:19:21.379 INFO:teuthology.orchestra.run.smithi089.stdout: 5 228433 9.19 MB/sec execute 480 sec latency 1092.914 ms 2024-09-21T04:19:22.265 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:19:22 smithi089 ceph-mon[64076]: osdmap e129: 8 total, 8 up, 8 in 2024-09-21T04:19:22.265 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:19:22 smithi089 ceph-mon[64076]: 2.1d starting backfill to osd.4 from (0'0,0'0] MAX to 110'9791 2024-09-21T04:19:22.379 INFO:teuthology.orchestra.run.smithi089.stdout: 5 228689 9.18 MB/sec execute 481 sec latency 600.707 ms 2024-09-21T04:19:22.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:19:22 smithi195 ceph-mon[49435]: osdmap e129: 8 total, 8 up, 8 in 2024-09-21T04:19:22.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:19:22 smithi195 ceph-mon[49435]: 2.1d starting backfill to osd.4 from (0'0,0'0] MAX to 110'9791 2024-09-21T04:19:23.291 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:19:23 smithi089 ceph-mon[64076]: pgmap v218: 97 pgs: 3 remapped+peering, 1 peering, 7 active+clean+remapped, 12 active+undersized, 23 active+undersized+degraded, 51 active+clean; 343 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 5.0 KiB/s rd, 4.2 MiB/s wr, 445 op/s; 237/1644 objects degraded (14.416%); 33/1644 objects misplaced (2.007%) 2024-09-21T04:19:23.291 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:19:23 smithi089 ceph-mon[64076]: osdmap e130: 8 total, 8 up, 8 in 2024-09-21T04:19:23.379 INFO:teuthology.orchestra.run.smithi089.stdout: 5 228990 9.18 MB/sec execute 482 sec latency 41.508 ms 2024-09-21T04:19:23.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:19:23 smithi195 ceph-mon[49435]: pgmap v218: 97 pgs: 3 remapped+peering, 1 peering, 7 active+clean+remapped, 12 active+undersized, 23 active+undersized+degraded, 51 active+clean; 343 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 5.0 KiB/s rd, 4.2 MiB/s wr, 445 op/s; 237/1644 objects degraded (14.416%); 33/1644 objects misplaced (2.007%) 2024-09-21T04:19:23.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:19:23 smithi195 ceph-mon[49435]: osdmap e130: 8 total, 8 up, 8 in 2024-09-21T04:19:24.296 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:19:24 smithi089 ceph-mon[64076]: osdmap e131: 8 total, 8 up, 8 in 2024-09-21T04:19:24.379 INFO:teuthology.orchestra.run.smithi089.stdout: 5 229311 9.18 MB/sec execute 483 sec latency 428.730 ms 2024-09-21T04:19:24.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:19:24 smithi195 ceph-mon[49435]: osdmap e131: 8 total, 8 up, 8 in 2024-09-21T04:19:25.363 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:19:25 smithi089 ceph-mon[64076]: pgmap v221: 97 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 3 active+undersized+remapped, 3 remapped+peering, 4 peering, 4 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 8 active+undersized, 5 active+undersized+degraded, 66 active+clean; 336 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 7.0 KiB/s rd, 4.4 MiB/s wr, 533 op/s; 169/1581 objects degraded (10.689%); 5/1581 objects misplaced (0.316%); 7.8 MiB/s, 69 keys/s, 33 objects/s recovering 2024-09-21T04:19:25.363 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:19:25 smithi089 ceph-mon[64076]: osdmap e132: 8 total, 8 up, 8 in 2024-09-21T04:19:25.363 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:19:25 smithi089 ceph-mon[64076]: Health check update: Degraded data redundancy: 169/1581 objects degraded (10.689%), 12 pgs degraded (PG_DEGRADED) 2024-09-21T04:19:25.363 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:19:25 smithi089 ceph-mon[64076]: osdmap e133: 8 total, 8 up, 8 in 2024-09-21T04:19:25.379 INFO:teuthology.orchestra.run.smithi089.stdout: 5 229704 9.18 MB/sec execute 484 sec latency 411.179 ms 2024-09-21T04:19:25.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:19:25 smithi195 ceph-mon[49435]: pgmap v221: 97 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 3 active+undersized+remapped, 3 remapped+peering, 4 peering, 4 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 8 active+undersized, 5 active+undersized+degraded, 66 active+clean; 336 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 7.0 KiB/s rd, 4.4 MiB/s wr, 533 op/s; 169/1581 objects degraded (10.689%); 5/1581 objects misplaced (0.316%); 7.8 MiB/s, 69 keys/s, 33 objects/s recovering 2024-09-21T04:19:25.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:19:25 smithi195 ceph-mon[49435]: osdmap e132: 8 total, 8 up, 8 in 2024-09-21T04:19:25.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:19:25 smithi195 ceph-mon[49435]: Health check update: Degraded data redundancy: 169/1581 objects degraded (10.689%), 12 pgs degraded (PG_DEGRADED) 2024-09-21T04:19:25.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:19:25 smithi195 ceph-mon[49435]: osdmap e133: 8 total, 8 up, 8 in 2024-09-21T04:19:26.379 INFO:teuthology.orchestra.run.smithi089.stdout: 5 229964 9.17 MB/sec execute 485 sec latency 952.165 ms 2024-09-21T04:19:27.149 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:19:26 smithi089 ceph-mon[64076]: osdmap e134: 8 total, 8 up, 8 in 2024-09-21T04:19:27.149 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:19:26 smithi089 ceph-mon[64076]: pgmap v225: 97 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 3 active+undersized+remapped, 3 peering, 4 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+undersized+degraded, 82 active+clean; 336 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 8.5 KiB/s rd, 3.9 MiB/s wr, 344 op/s; 128/1548 objects degraded (8.269%); 5/1548 objects misplaced (0.323%); 10 MiB/s, 71 keys/s, 53 objects/s recovering 2024-09-21T04:19:27.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:19:26 smithi195 ceph-mon[49435]: osdmap e134: 8 total, 8 up, 8 in 2024-09-21T04:19:27.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:19:26 smithi195 ceph-mon[49435]: pgmap v225: 97 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 3 active+undersized+remapped, 3 peering, 4 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+undersized+degraded, 82 active+clean; 336 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 8.5 KiB/s rd, 3.9 MiB/s wr, 344 op/s; 128/1548 objects degraded (8.269%); 5/1548 objects misplaced (0.323%); 10 MiB/s, 71 keys/s, 53 objects/s recovering 2024-09-21T04:19:27.379 INFO:teuthology.orchestra.run.smithi089.stdout: 5 230233 9.17 MB/sec execute 486 sec latency 969.270 ms 2024-09-21T04:19:27.914 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:19:27 smithi089 ceph-mon[64076]: osdmap e135: 8 total, 8 up, 8 in 2024-09-21T04:19:28.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:19:27 smithi195 ceph-mon[49435]: osdmap e135: 8 total, 8 up, 8 in 2024-09-21T04:19:28.379 INFO:teuthology.orchestra.run.smithi089.stdout: 5 230541 9.18 MB/sec execute 487 sec latency 44.698 ms 2024-09-21T04:19:29.186 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:19:28 smithi089 ceph-mon[64076]: pgmap v227: 97 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 3 peering, 1 active+remapped+backfill_wait, 1 active+undersized+degraded, 91 active+clean; 338 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 8.7 KiB/s rd, 2.8 MiB/s wr, 156 op/s; 42/1572 objects degraded (2.672%); 5/1572 objects misplaced (0.318%); 2.2 MiB/s, 39 objects/s recovering 2024-09-21T04:19:29.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:19:28 smithi195 ceph-mon[49435]: pgmap v227: 97 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 3 peering, 1 active+remapped+backfill_wait, 1 active+undersized+degraded, 91 active+clean; 338 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 8.7 KiB/s rd, 2.8 MiB/s wr, 156 op/s; 42/1572 objects degraded (2.672%); 5/1572 objects misplaced (0.318%); 2.2 MiB/s, 39 objects/s recovering 2024-09-21T04:19:29.379 INFO:teuthology.orchestra.run.smithi089.stdout: 5 230950 9.19 MB/sec execute 488 sec latency 47.832 ms 2024-09-21T04:19:29.944 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:19:29 smithi089 ceph-mon[64076]: Health check update: Degraded data redundancy: 42/1572 objects degraded (2.672%), 2 pgs degraded (PG_DEGRADED) 2024-09-21T04:19:30.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:19:29 smithi195 ceph-mon[49435]: Health check update: Degraded data redundancy: 42/1572 objects degraded (2.672%), 2 pgs degraded (PG_DEGRADED) 2024-09-21T04:19:30.379 INFO:teuthology.orchestra.run.smithi089.stdout: 5 231298 9.19 MB/sec execute 489 sec latency 64.958 ms 2024-09-21T04:19:30.972 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:19:30 smithi089 ceph-mon[64076]: pgmap v228: 97 pgs: 97 active+clean; 339 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 3.8 MiB/s wr, 401 op/s; 2.9 MiB/s, 1 keys/s, 28 objects/s recovering 2024-09-21T04:19:31.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:19:30 smithi195 ceph-mon[49435]: pgmap v228: 97 pgs: 97 active+clean; 339 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 3.8 MiB/s wr, 401 op/s; 2.9 MiB/s, 1 keys/s, 28 objects/s recovering 2024-09-21T04:19:31.379 INFO:teuthology.orchestra.run.smithi089.stdout: 5 231483 9.18 MB/sec execute 490 sec latency 65.365 ms 2024-09-21T04:19:32.012 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:19:31 smithi089 ceph-mon[64076]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 42/1572 objects degraded (2.672%), 2 pgs degraded) 2024-09-21T04:19:32.012 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:19:31 smithi089 ceph-mon[64076]: Cluster is now healthy 2024-09-21T04:19:32.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:19:31 smithi195 ceph-mon[49435]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 42/1572 objects degraded (2.672%), 2 pgs degraded) 2024-09-21T04:19:32.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:19:31 smithi195 ceph-mon[49435]: Cluster is now healthy 2024-09-21T04:19:32.379 INFO:teuthology.orchestra.run.smithi089.stdout: 5 231739 9.17 MB/sec execute 491 sec latency 66.655 ms 2024-09-21T04:19:33.028 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:19:32 smithi089 ceph-mon[64076]: pgmap v229: 97 pgs: 97 active+clean; 340 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 9.0 KiB/s rd, 3.3 MiB/s wr, 338 op/s; 2.4 MiB/s, 0 keys/s, 23 objects/s recovering 2024-09-21T04:19:33.028 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:19:32 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:19:33.028 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:19:32 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T04:19:33.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:19:32 smithi195 ceph-mon[49435]: pgmap v229: 97 pgs: 97 active+clean; 340 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 9.0 KiB/s rd, 3.3 MiB/s wr, 338 op/s; 2.4 MiB/s, 0 keys/s, 23 objects/s recovering 2024-09-21T04:19:33.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:19:32 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:19:33.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:19:32 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T04:19:33.379 INFO:teuthology.orchestra.run.smithi089.stdout: 5 232054 9.16 MB/sec execute 492 sec latency 46.293 ms 2024-09-21T04:19:34.034 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:19:33 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16} : dispatch 2024-09-21T04:19:34.271 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:19:33 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16} : dispatch 2024-09-21T04:19:34.380 INFO:teuthology.orchestra.run.smithi089.stdout: 5 232347 9.16 MB/sec execute 493 sec latency 54.278 ms 2024-09-21T04:19:35.055 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:19:34 smithi089 ceph-mon[64076]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-09-21T04:19:35.055 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:19:34 smithi089 ceph-mon[64076]: Upgrade: osd.5 is safe to restart 2024-09-21T04:19:35.055 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:19:34 smithi089 ceph-mon[64076]: pgmap v230: 97 pgs: 97 active+clean; 337 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 3.8 KiB/s rd, 5.2 MiB/s wr, 517 op/s; 1.0 MiB/s, 0 keys/s, 17 objects/s recovering 2024-09-21T04:19:35.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:19:34 smithi195 ceph-mon[49435]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-09-21T04:19:35.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:19:34 smithi195 ceph-mon[49435]: Upgrade: osd.5 is safe to restart 2024-09-21T04:19:35.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:19:34 smithi195 ceph-mon[49435]: pgmap v230: 97 pgs: 97 active+clean; 337 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 3.8 KiB/s rd, 5.2 MiB/s wr, 517 op/s; 1.0 MiB/s, 0 keys/s, 17 objects/s recovering 2024-09-21T04:19:35.379 INFO:teuthology.orchestra.run.smithi089.stdout: 5 232664 9.16 MB/sec execute 494 sec latency 43.517 ms 2024-09-21T04:19:36.380 INFO:teuthology.orchestra.run.smithi089.stdout: 5 232978 9.15 MB/sec execute 495 sec latency 66.124 ms 2024-09-21T04:19:36.582 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:19:36 smithi089 ceph-mon[64076]: Upgrade: Updating osd.5 2024-09-21T04:19:36.582 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:19:36 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:19:36.582 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:19:36 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "auth get", "entity": "osd.5"} : dispatch 2024-09-21T04:19:36.582 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:19:36 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T04:19:36.582 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:19:36 smithi089 ceph-mon[64076]: Deploying daemon osd.5 on smithi195 2024-09-21T04:19:36.582 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:19:36 smithi089 ceph-mon[64076]: pgmap v231: 97 pgs: 97 active+clean; 337 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 7.4 KiB/s rd, 4.5 MiB/s wr, 432 op/s; 868 KiB/s, 0 keys/s, 13 objects/s recovering 2024-09-21T04:19:36.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:19:36 smithi195 ceph-mon[49435]: Upgrade: Updating osd.5 2024-09-21T04:19:36.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:19:36 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:19:36.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:19:36 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "auth get", "entity": "osd.5"} : dispatch 2024-09-21T04:19:36.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:19:36 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T04:19:36.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:19:36 smithi195 ceph-mon[49435]: Deploying daemon osd.5 on smithi195 2024-09-21T04:19:36.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:19:36 smithi195 ceph-mon[49435]: pgmap v231: 97 pgs: 97 active+clean; 337 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 7.4 KiB/s rd, 4.5 MiB/s wr, 432 op/s; 868 KiB/s, 0 keys/s, 13 objects/s recovering 2024-09-21T04:19:37.380 INFO:teuthology.orchestra.run.smithi089.stdout: 5 233545 9.16 MB/sec execute 496 sec latency 25.879 ms 2024-09-21T04:19:38.380 INFO:teuthology.orchestra.run.smithi089.stdout: 5 233990 9.18 MB/sec execute 497 sec latency 36.016 ms 2024-09-21T04:19:38.610 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:19:38 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:19:38.610 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:19:38 smithi195 ceph-mon[49435]: pgmap v232: 97 pgs: 97 active+clean; 340 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 6.8 KiB/s rd, 5.6 MiB/s wr, 594 op/s; 782 KiB/s, 0 keys/s, 12 objects/s recovering 2024-09-21T04:19:38.631 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:19:38 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:19:38.631 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:19:38 smithi089 ceph-mon[64076]: pgmap v232: 97 pgs: 97 active+clean; 340 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 6.8 KiB/s rd, 5.6 MiB/s wr, 594 op/s; 782 KiB/s, 0 keys/s, 12 objects/s recovering 2024-09-21T04:19:39.109 INFO:journalctl@ceph.osd.5.smithi195.stdout:Sep 21 04:19:38 smithi195 systemd[1]: Stopping Ceph osd.5 for 1b9b79fc-77ce-11ef-baf5-efdc52797490... 2024-09-21T04:19:39.380 INFO:teuthology.orchestra.run.smithi089.stdout: 5 234390 9.19 MB/sec execute 498 sec latency 66.517 ms 2024-09-21T04:19:39.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:19:39 smithi195 ceph-mon[49435]: osd.5 marked itself down and dead 2024-09-21T04:19:39.609 INFO:journalctl@ceph.osd.5.smithi195.stdout:Sep 21 04:19:39 smithi195 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-5[32992]: 2024-09-21T04:19:39.233+0000 7fdee0db6700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.5 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-21T04:19:39.609 INFO:journalctl@ceph.osd.5.smithi195.stdout:Sep 21 04:19:39 smithi195 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-5[32992]: 2024-09-21T04:19:39.233+0000 7fdee0db6700 -1 osd.5 135 *** Got signal Terminated *** 2024-09-21T04:19:39.609 INFO:journalctl@ceph.osd.5.smithi195.stdout:Sep 21 04:19:39 smithi195 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-5[32992]: 2024-09-21T04:19:39.233+0000 7fdee0db6700 -1 osd.5 135 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-21T04:19:39.673 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:19:39 smithi089 ceph-mon[64076]: osd.5 marked itself down and dead 2024-09-21T04:19:40.354 INFO:journalctl@ceph.osd.5.smithi195.stdout:Sep 21 04:19:40 smithi195 podman[54917]: 2024-09-21 04:19:40.09762553 +0000 UTC m=+0.976869896 container died cecfd6ff1eee72a931112b9ca0c3749f928df2a0bd5c0531de61a2f6fe329fe6 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-5, RELEASE=HEAD, io.k8s.display-name=CentOS Stream 8, GIT_BRANCH=HEAD, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., version=8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, architecture=x86_64, distribution-scope=public, vendor=Red Hat, Inc., io.buildah.version=1.19.8, maintainer=Guillaume Abrioux , name=centos-stream, ceph=True, GIT_REPO=https://github.com/ceph/ceph-container.git, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, build-date=2022-05-03T08:36:31.336870, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, com.redhat.component=centos-stream-container, CEPH_POINT_RELEASE=-17.2.0, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.openshift.tags=base centos centos-stream, io.openshift.expose-services=, vcs-type=git, GIT_CLEAN=True, release=754) 2024-09-21T04:19:40.380 INFO:teuthology.orchestra.run.smithi089.stdout: 5 234582 9.18 MB/sec execute 499 sec latency 924.514 ms 2024-09-21T04:19:40.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:19:40 smithi195 ceph-mon[49435]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-21T04:19:40.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:19:40 smithi195 ceph-mon[49435]: osdmap e136: 8 total, 7 up, 8 in 2024-09-21T04:19:40.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:19:40 smithi195 ceph-mon[49435]: pgmap v234: 97 pgs: 12 stale+active+clean, 85 active+clean; 342 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 7.4 KiB/s rd, 5.1 MiB/s wr, 477 op/s 2024-09-21T04:19:40.610 INFO:journalctl@ceph.osd.5.smithi195.stdout:Sep 21 04:19:40 smithi195 podman[54917]: 2024-09-21 04:19:40.442631745 +0000 UTC m=+1.321876106 container cleanup cecfd6ff1eee72a931112b9ca0c3749f928df2a0bd5c0531de61a2f6fe329fe6 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-5, io.openshift.expose-services=, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, architecture=x86_64, vcs-type=git, GIT_CLEAN=True, version=8, io.openshift.tags=base centos centos-stream, CEPH_POINT_RELEASE=-17.2.0, release=754, com.redhat.component=centos-stream-container, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, build-date=2022-05-03T08:36:31.336870, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_REPO=https://github.com/ceph/ceph-container.git, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.k8s.display-name=CentOS Stream 8, RELEASE=HEAD, ceph=True, distribution-scope=public, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, name=centos-stream, vendor=Red Hat, Inc., io.buildah.version=1.19.8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac) 2024-09-21T04:19:40.610 INFO:journalctl@ceph.osd.5.smithi195.stdout:Sep 21 04:19:40 smithi195 bash[54917]: ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-5 2024-09-21T04:19:40.695 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:19:40 smithi089 ceph-mon[64076]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-21T04:19:40.695 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:19:40 smithi089 ceph-mon[64076]: osdmap e136: 8 total, 7 up, 8 in 2024-09-21T04:19:40.695 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:19:40 smithi089 ceph-mon[64076]: pgmap v234: 97 pgs: 12 stale+active+clean, 85 active+clean; 342 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 7.4 KiB/s rd, 5.1 MiB/s wr, 477 op/s 2024-09-21T04:19:41.109 INFO:journalctl@ceph.osd.5.smithi195.stdout:Sep 21 04:19:40 smithi195 podman[54941]: 2024-09-21 04:19:40.669743632 +0000 UTC m=+0.569179956 container remove cecfd6ff1eee72a931112b9ca0c3749f928df2a0bd5c0531de61a2f6fe329fe6 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-5, release=754, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, architecture=x86_64, build-date=2022-05-03T08:36:31.336870, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_BRANCH=HEAD, GIT_CLEAN=True, RELEASE=HEAD, com.redhat.component=centos-stream-container, version=8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, distribution-scope=public, name=centos-stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vendor=Red Hat, Inc., ceph=True, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.buildah.version=1.19.8, maintainer=Guillaume Abrioux , vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.k8s.display-name=CentOS Stream 8, vcs-type=git, io.openshift.tags=base centos centos-stream, GIT_REPO=https://github.com/ceph/ceph-container.git, CEPH_POINT_RELEASE=-17.2.0, io.openshift.expose-services=) 2024-09-21T04:19:41.380 INFO:teuthology.orchestra.run.smithi089.stdout: 5 234960 9.18 MB/sec execute 500 sec latency 1023.346 ms 2024-09-21T04:19:41.451 INFO:journalctl@ceph.osd.5.smithi195.stdout:Sep 21 04:19:41 smithi195 podman[55035]: 2024-09-21 04:19:41.060584863 +0000 UTC m=+0.018525238 image pull 48ed0a2cf89d6d0371a2c6185a5869a747265f6abd5ac2a74f00cc32d5628721 quay.ceph.io/ceph-ci/ceph@sha256:2195e5f178be0a9390728e2a14088236d8fe6f82635f5c84418fe37a04457e00 2024-09-21T04:19:41.452 INFO:journalctl@ceph.osd.5.smithi195.stdout:Sep 21 04:19:41 smithi195 podman[55035]: 2024-09-21 04:19:41.195901576 +0000 UTC m=+0.153841936 container create 96a10ec1f29c288ce445f0f4a92c223555d4a72c582107725b7cf6f02b01bfd2 (image=quay.ceph.io/ceph-ci/ceph@sha256:2195e5f178be0a9390728e2a14088236d8fe6f82635f5c84418fe37a04457e00, name=ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-5-deactivate, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, GIT_BRANCH=HEAD, ceph=True, RELEASE=wip-guits-main-2024-09-20-2310-f36283e, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux ) 2024-09-21T04:19:41.458 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:19:41 smithi089 ceph-mon[64076]: osdmap e137: 8 total, 7 up, 8 in 2024-09-21T04:19:41.705 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:19:41 smithi195 ceph-mon[49435]: osdmap e137: 8 total, 7 up, 8 in 2024-09-21T04:19:41.705 INFO:journalctl@ceph.osd.5.smithi195.stdout:Sep 21 04:19:41 smithi195 podman[55035]: 2024-09-21 04:19:41.490883967 +0000 UTC m=+0.448824344 container init 96a10ec1f29c288ce445f0f4a92c223555d4a72c582107725b7cf6f02b01bfd2 (image=quay.ceph.io/ceph-ci/ceph@sha256:2195e5f178be0a9390728e2a14088236d8fe6f82635f5c84418fe37a04457e00, name=ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-5-deactivate, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, ceph=True, GIT_CLEAN=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-guits-main-2024-09-20-2310-f36283e, GIT_BRANCH=HEAD) 2024-09-21T04:19:41.705 INFO:journalctl@ceph.osd.5.smithi195.stdout:Sep 21 04:19:41 smithi195 podman[55035]: 2024-09-21 04:19:41.494783934 +0000 UTC m=+0.452724302 container start 96a10ec1f29c288ce445f0f4a92c223555d4a72c582107725b7cf6f02b01bfd2 (image=quay.ceph.io/ceph-ci/ceph@sha256:2195e5f178be0a9390728e2a14088236d8fe6f82635f5c84418fe37a04457e00, name=ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-5-deactivate, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, org.label-schema.build-date=20240731, RELEASE=wip-guits-main-2024-09-20-2310-f36283e, org.label-schema.schema-version=1.0, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, ceph=True, GIT_BRANCH=HEAD, io.buildah.version=1.37.2) 2024-09-21T04:19:41.705 INFO:journalctl@ceph.osd.5.smithi195.stdout:Sep 21 04:19:41 smithi195 podman[55035]: 2024-09-21 04:19:41.582516198 +0000 UTC m=+0.540456586 container attach 96a10ec1f29c288ce445f0f4a92c223555d4a72c582107725b7cf6f02b01bfd2 (image=quay.ceph.io/ceph-ci/ceph@sha256:2195e5f178be0a9390728e2a14088236d8fe6f82635f5c84418fe37a04457e00, name=ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-5-deactivate, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, RELEASE=wip-guits-main-2024-09-20-2310-f36283e, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, ceph=True) 2024-09-21T04:19:42.071 INFO:journalctl@ceph.osd.5.smithi195.stdout:Sep 21 04:19:41 smithi195 podman[55035]: 2024-09-21 04:19:41.701223485 +0000 UTC m=+0.659163853 container died 96a10ec1f29c288ce445f0f4a92c223555d4a72c582107725b7cf6f02b01bfd2 (image=quay.ceph.io/ceph-ci/ceph@sha256:2195e5f178be0a9390728e2a14088236d8fe6f82635f5c84418fe37a04457e00, name=ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-5-deactivate, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, RELEASE=wip-guits-main-2024-09-20-2310-f36283e, org.label-schema.license=GPLv2, ceph=True, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0) 2024-09-21T04:19:42.380 INFO:teuthology.orchestra.run.smithi089.stdout: 5 235305 9.18 MB/sec execute 501 sec latency 58.333 ms 2024-09-21T04:19:42.461 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:19:42 smithi089 ceph-mon[64076]: pgmap v236: 97 pgs: 2 activating+undersized, 11 stale+active+clean, 1 active+undersized+degraded, 83 active+clean; 339 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 9.4 KiB/s rd, 3.9 MiB/s wr, 402 op/s; 12/1695 objects degraded (0.708%) 2024-09-21T04:19:42.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:19:42 smithi195 ceph-mon[49435]: pgmap v236: 97 pgs: 2 activating+undersized, 11 stale+active+clean, 1 active+undersized+degraded, 83 active+clean; 339 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 9.4 KiB/s rd, 3.9 MiB/s wr, 402 op/s; 12/1695 objects degraded (0.708%) 2024-09-21T04:19:42.609 INFO:journalctl@ceph.osd.5.smithi195.stdout:Sep 21 04:19:42 smithi195 podman[55058]: 2024-09-21 04:19:42.345340441 +0000 UTC m=+0.631879222 container remove 96a10ec1f29c288ce445f0f4a92c223555d4a72c582107725b7cf6f02b01bfd2 (image=quay.ceph.io/ceph-ci/ceph@sha256:2195e5f178be0a9390728e2a14088236d8fe6f82635f5c84418fe37a04457e00, name=ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-5-deactivate, RELEASE=wip-guits-main-2024-09-20-2310-f36283e, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, GIT_CLEAN=True, ceph=True, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-21T04:19:42.609 INFO:journalctl@ceph.osd.5.smithi195.stdout:Sep 21 04:19:42 smithi195 systemd[1]: ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490@osd.5.service: Deactivated successfully. 2024-09-21T04:19:42.610 INFO:journalctl@ceph.osd.5.smithi195.stdout:Sep 21 04:19:42 smithi195 systemd[1]: Stopped Ceph osd.5 for 1b9b79fc-77ce-11ef-baf5-efdc52797490. 2024-09-21T04:19:42.610 INFO:journalctl@ceph.osd.5.smithi195.stdout:Sep 21 04:19:42 smithi195 systemd[1]: ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490@osd.5.service: Consumed 4min 44.177s CPU time. 2024-09-21T04:19:42.610 INFO:journalctl@ceph.osd.5.smithi195.stdout:Sep 21 04:19:42 smithi195 systemd[1]: Starting Ceph osd.5 for 1b9b79fc-77ce-11ef-baf5-efdc52797490... 2024-09-21T04:19:43.093 INFO:journalctl@ceph.osd.5.smithi195.stdout:Sep 21 04:19:42 smithi195 podman[55158]: 2024-09-21 04:19:42.690213034 +0000 UTC m=+0.016347995 image pull 48ed0a2cf89d6d0371a2c6185a5869a747265f6abd5ac2a74f00cc32d5628721 quay.ceph.io/ceph-ci/ceph@sha256:2195e5f178be0a9390728e2a14088236d8fe6f82635f5c84418fe37a04457e00 2024-09-21T04:19:43.093 INFO:journalctl@ceph.osd.5.smithi195.stdout:Sep 21 04:19:42 smithi195 podman[55158]: 2024-09-21 04:19:42.821446683 +0000 UTC m=+0.147581644 container create 0e8f7d2ab0e42054316767d1d6ec2a451a1a7f5a70bdf8700942c6c81f06c405 (image=quay.ceph.io/ceph-ci/ceph@sha256:2195e5f178be0a9390728e2a14088236d8fe6f82635f5c84418fe37a04457e00, name=ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-5-activate, RELEASE=wip-guits-main-2024-09-20-2310-f36283e, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, ceph=True, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731) 2024-09-21T04:19:43.093 INFO:journalctl@ceph.osd.5.smithi195.stdout:Sep 21 04:19:43 smithi195 podman[55158]: 2024-09-21 04:19:43.091416483 +0000 UTC m=+0.417551452 container init 0e8f7d2ab0e42054316767d1d6ec2a451a1a7f5a70bdf8700942c6c81f06c405 (image=quay.ceph.io/ceph-ci/ceph@sha256:2195e5f178be0a9390728e2a14088236d8fe6f82635f5c84418fe37a04457e00, name=ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-5-activate, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , RELEASE=wip-guits-main-2024-09-20-2310-f36283e, GIT_CLEAN=True, org.label-schema.build-date=20240731) 2024-09-21T04:19:43.359 INFO:journalctl@ceph.osd.5.smithi195.stdout:Sep 21 04:19:43 smithi195 podman[55158]: 2024-09-21 04:19:43.094952151 +0000 UTC m=+0.421087117 container start 0e8f7d2ab0e42054316767d1d6ec2a451a1a7f5a70bdf8700942c6c81f06c405 (image=quay.ceph.io/ceph-ci/ceph@sha256:2195e5f178be0a9390728e2a14088236d8fe6f82635f5c84418fe37a04457e00, name=ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-5-activate, org.label-schema.license=GPLv2, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, RELEASE=wip-guits-main-2024-09-20-2310-f36283e, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_BRANCH=HEAD, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.schema-version=1.0) 2024-09-21T04:19:43.359 INFO:journalctl@ceph.osd.5.smithi195.stdout:Sep 21 04:19:43 smithi195 podman[55158]: 2024-09-21 04:19:43.1915385 +0000 UTC m=+0.517673479 container attach 0e8f7d2ab0e42054316767d1d6ec2a451a1a7f5a70bdf8700942c6c81f06c405 (image=quay.ceph.io/ceph-ci/ceph@sha256:2195e5f178be0a9390728e2a14088236d8fe6f82635f5c84418fe37a04457e00, name=ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-5-activate, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, ceph=True, GIT_CLEAN=True, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, RELEASE=wip-guits-main-2024-09-20-2310-f36283e, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-21T04:19:43.359 INFO:journalctl@ceph.osd.5.smithi195.stdout:Sep 21 04:19:43 smithi195 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-5-activate[55174]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T04:19:43.359 INFO:journalctl@ceph.osd.5.smithi195.stdout:Sep 21 04:19:43 smithi195 bash[55158]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T04:19:43.360 INFO:journalctl@ceph.osd.5.smithi195.stdout:Sep 21 04:19:43 smithi195 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-5-activate[55174]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T04:19:43.360 INFO:journalctl@ceph.osd.5.smithi195.stdout:Sep 21 04:19:43 smithi195 bash[55158]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T04:19:43.380 INFO:teuthology.orchestra.run.smithi089.stdout: 5 235758 9.18 MB/sec execute 502 sec latency 24.686 ms 2024-09-21T04:19:43.656 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:19:43 smithi089 ceph-mon[64076]: Health check failed: Degraded data redundancy: 12/1695 objects degraded (0.708%), 1 pg degraded (PG_DEGRADED) 2024-09-21T04:19:43.859 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:19:43 smithi195 ceph-mon[49435]: Health check failed: Degraded data redundancy: 12/1695 objects degraded (0.708%), 1 pg degraded (PG_DEGRADED) 2024-09-21T04:19:44.380 INFO:teuthology.orchestra.run.smithi089.stdout: 5 236079 9.18 MB/sec execute 503 sec latency 33.292 ms 2024-09-21T04:19:44.668 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:19:44 smithi089 ceph-mon[64076]: pgmap v237: 97 pgs: 6 active+undersized, 2 activating+undersized, 3 stale+active+clean, 18 active+undersized+degraded, 68 active+clean; 340 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 4.0 KiB/s rd, 6.0 MiB/s wr, 597 op/s; 167/1704 objects degraded (9.800%) 2024-09-21T04:19:44.859 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:19:44 smithi195 ceph-mon[49435]: pgmap v237: 97 pgs: 6 active+undersized, 2 activating+undersized, 3 stale+active+clean, 18 active+undersized+degraded, 68 active+clean; 340 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 4.0 KiB/s rd, 6.0 MiB/s wr, 597 op/s; 167/1704 objects degraded (9.800%) 2024-09-21T04:19:45.380 INFO:teuthology.orchestra.run.smithi089.stdout: 5 236475 9.19 MB/sec execute 504 sec latency 49.820 ms 2024-09-21T04:19:46.380 INFO:teuthology.orchestra.run.smithi089.stdout: 5 236960 9.18 MB/sec execute 505 sec latency 45.965 ms 2024-09-21T04:19:46.508 INFO:journalctl@ceph.osd.5.smithi195.stdout:Sep 21 04:19:46 smithi195 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-5-activate[55174]: --> Failed to activate via raw: did not find any matching OSD to activate 2024-09-21T04:19:46.508 INFO:journalctl@ceph.osd.5.smithi195.stdout:Sep 21 04:19:46 smithi195 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-5-activate[55174]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T04:19:46.508 INFO:journalctl@ceph.osd.5.smithi195.stdout:Sep 21 04:19:46 smithi195 bash[55158]: --> Failed to activate via raw: did not find any matching OSD to activate 2024-09-21T04:19:46.508 INFO:journalctl@ceph.osd.5.smithi195.stdout:Sep 21 04:19:46 smithi195 bash[55158]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T04:19:46.509 INFO:journalctl@ceph.osd.5.smithi195.stdout:Sep 21 04:19:46 smithi195 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-5-activate[55174]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T04:19:46.509 INFO:journalctl@ceph.osd.5.smithi195.stdout:Sep 21 04:19:46 smithi195 bash[55158]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T04:19:46.509 INFO:journalctl@ceph.osd.5.smithi195.stdout:Sep 21 04:19:46 smithi195 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-5-activate[55174]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2024-09-21T04:19:46.509 INFO:journalctl@ceph.osd.5.smithi195.stdout:Sep 21 04:19:46 smithi195 bash[55158]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2024-09-21T04:19:46.509 INFO:journalctl@ceph.osd.5.smithi195.stdout:Sep 21 04:19:46 smithi195 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-5-activate[55174]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_3 --path /var/lib/ceph/osd/ceph-5 --no-mon-config 2024-09-21T04:19:46.509 INFO:journalctl@ceph.osd.5.smithi195.stdout:Sep 21 04:19:46 smithi195 bash[55158]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_3 --path /var/lib/ceph/osd/ceph-5 --no-mon-config 2024-09-21T04:19:46.509 INFO:journalctl@ceph.osd.5.smithi195.stdout:Sep 21 04:19:46 smithi195 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-5-activate[55174]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_3 /var/lib/ceph/osd/ceph-5/block 2024-09-21T04:19:46.509 INFO:journalctl@ceph.osd.5.smithi195.stdout:Sep 21 04:19:46 smithi195 bash[55158]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_3 /var/lib/ceph/osd/ceph-5/block 2024-09-21T04:19:46.856 INFO:journalctl@ceph.osd.5.smithi195.stdout:Sep 21 04:19:46 smithi195 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-5-activate[55174]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-5/block 2024-09-21T04:19:46.856 INFO:journalctl@ceph.osd.5.smithi195.stdout:Sep 21 04:19:46 smithi195 bash[55158]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-5/block 2024-09-21T04:19:46.856 INFO:journalctl@ceph.osd.5.smithi195.stdout:Sep 21 04:19:46 smithi195 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-5-activate[55174]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-09-21T04:19:46.857 INFO:journalctl@ceph.osd.5.smithi195.stdout:Sep 21 04:19:46 smithi195 bash[55158]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-09-21T04:19:46.857 INFO:journalctl@ceph.osd.5.smithi195.stdout:Sep 21 04:19:46 smithi195 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-5-activate[55174]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2024-09-21T04:19:46.857 INFO:journalctl@ceph.osd.5.smithi195.stdout:Sep 21 04:19:46 smithi195 bash[55158]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2024-09-21T04:19:46.857 INFO:journalctl@ceph.osd.5.smithi195.stdout:Sep 21 04:19:46 smithi195 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-5-activate[55174]: --> ceph-volume lvm activate successful for osd ID: 5 2024-09-21T04:19:46.857 INFO:journalctl@ceph.osd.5.smithi195.stdout:Sep 21 04:19:46 smithi195 bash[55158]: --> ceph-volume lvm activate successful for osd ID: 5 2024-09-21T04:19:46.857 INFO:journalctl@ceph.osd.5.smithi195.stdout:Sep 21 04:19:46 smithi195 podman[55158]: 2024-09-21 04:19:46.544236694 +0000 UTC m=+3.870371665 container died 0e8f7d2ab0e42054316767d1d6ec2a451a1a7f5a70bdf8700942c6c81f06c405 (image=quay.ceph.io/ceph-ci/ceph@sha256:2195e5f178be0a9390728e2a14088236d8fe6f82635f5c84418fe37a04457e00, name=ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-5-activate, RELEASE=wip-guits-main-2024-09-20-2310-f36283e, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, GIT_CLEAN=True) 2024-09-21T04:19:47.109 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:19:47 smithi195 ceph-mon[49435]: pgmap v238: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 339 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 9.5 KiB/s rd, 4.7 MiB/s wr, 423 op/s; 186/1710 objects degraded (10.877%) 2024-09-21T04:19:47.182 INFO:teuthology.orchestra.run.smithi089.stdout:true 2024-09-21T04:19:47.213 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:19:47 smithi089 ceph-mon[64076]: pgmap v238: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 339 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 9.5 KiB/s rd, 4.7 MiB/s wr, 423 op/s; 186/1710 objects degraded (10.877%) 2024-09-21T04:19:47.380 INFO:teuthology.orchestra.run.smithi089.stdout: 5 237252 9.18 MB/sec execute 506 sec latency 55.225 ms 2024-09-21T04:19:47.416 INFO:journalctl@ceph.osd.5.smithi195.stdout:Sep 21 04:19:47 smithi195 podman[55386]: 2024-09-21 04:19:47.155015276 +0000 UTC m=+0.603916562 container remove 0e8f7d2ab0e42054316767d1d6ec2a451a1a7f5a70bdf8700942c6c81f06c405 (image=quay.ceph.io/ceph-ci/ceph@sha256:2195e5f178be0a9390728e2a14088236d8fe6f82635f5c84418fe37a04457e00, name=ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-5-activate, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, maintainer=Guillaume Abrioux , RELEASE=wip-guits-main-2024-09-20-2310-f36283e, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-21T04:19:47.416 INFO:journalctl@ceph.osd.5.smithi195.stdout:Sep 21 04:19:47 smithi195 podman[55446]: 2024-09-21 04:19:47.28711502 +0000 UTC m=+0.023097822 image pull 48ed0a2cf89d6d0371a2c6185a5869a747265f6abd5ac2a74f00cc32d5628721 quay.ceph.io/ceph-ci/ceph@sha256:2195e5f178be0a9390728e2a14088236d8fe6f82635f5c84418fe37a04457e00 2024-09-21T04:19:47.416 INFO:journalctl@ceph.osd.5.smithi195.stdout:Sep 21 04:19:47 smithi195 podman[55446]: 2024-09-21 04:19:47.414503181 +0000 UTC m=+0.150485975 container create 41de0beb713dabd6c82bbde19afc417f126e268c6ec14b757a87940c679e7f45 (image=quay.ceph.io/ceph-ci/ceph@sha256:2195e5f178be0a9390728e2a14088236d8fe6f82635f5c84418fe37a04457e00, name=ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-5, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, ceph=True, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, RELEASE=wip-guits-main-2024-09-20-2310-f36283e) 2024-09-21T04:19:48.056 INFO:journalctl@ceph.osd.5.smithi195.stdout:Sep 21 04:19:47 smithi195 podman[55446]: 2024-09-21 04:19:47.709356749 +0000 UTC m=+0.445339563 container init 41de0beb713dabd6c82bbde19afc417f126e268c6ec14b757a87940c679e7f45 (image=quay.ceph.io/ceph-ci/ceph@sha256:2195e5f178be0a9390728e2a14088236d8fe6f82635f5c84418fe37a04457e00, name=ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-5, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, RELEASE=wip-guits-main-2024-09-20-2310-f36283e, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-21T04:19:48.056 INFO:journalctl@ceph.osd.5.smithi195.stdout:Sep 21 04:19:47 smithi195 podman[55446]: 2024-09-21 04:19:47.714452421 +0000 UTC m=+0.450435221 container start 41de0beb713dabd6c82bbde19afc417f126e268c6ec14b757a87940c679e7f45 (image=quay.ceph.io/ceph-ci/ceph@sha256:2195e5f178be0a9390728e2a14088236d8fe6f82635f5c84418fe37a04457e00, name=ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-5, org.label-schema.license=GPLv2, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, RELEASE=wip-guits-main-2024-09-20-2310-f36283e, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23) 2024-09-21T04:19:48.056 INFO:journalctl@ceph.osd.5.smithi195.stdout:Sep 21 04:19:47 smithi195 ceph-osd[55464]: -- 172.21.15.195:0/2244861335 <== mon.1 v2:172.21.15.195:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55ccd43781e0 con 0x55ccd4362000 2024-09-21T04:19:48.057 INFO:journalctl@ceph.osd.5.smithi195.stdout:Sep 21 04:19:47 smithi195 bash[55446]: 41de0beb713dabd6c82bbde19afc417f126e268c6ec14b757a87940c679e7f45 2024-09-21T04:19:48.057 INFO:journalctl@ceph.osd.5.smithi195.stdout:Sep 21 04:19:47 smithi195 systemd[1]: Started Ceph osd.5 for 1b9b79fc-77ce-11ef-baf5-efdc52797490. 2024-09-21T04:19:48.231 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:19:48 smithi089 ceph-mon[64076]: from='client.34388 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T04:19:48.231 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:19:48 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:19:48.231 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:19:48 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T04:19:48.231 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:19:48 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:19:48.231 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:19:48 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:19:48.231 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:19:48 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T04:19:48.245 INFO:teuthology.orchestra.run.smithi089.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-21T04:19:48.245 INFO:teuthology.orchestra.run.smithi089.stdout:alertmanager.smithi089 smithi089 *:9093,9094 running (7m) 74s ago 16m 17.9M - 0.27.0 11f11916f8cd 65dfcbcafee1 2024-09-21T04:19:48.245 INFO:teuthology.orchestra.run.smithi089.stdout:crash.smithi089 smithi089 running (4m) 74s ago 16m 6891k - 19.3.0-5103-gf36283e7 48ed0a2cf89d 6c415a5e173e 2024-09-21T04:19:48.245 INFO:teuthology.orchestra.run.smithi089.stdout:crash.smithi195 smithi195 running (4m) 34s ago 16m 6891k - 19.3.0-5103-gf36283e7 48ed0a2cf89d 78ce05a4e9ab 2024-09-21T04:19:48.245 INFO:teuthology.orchestra.run.smithi089.stdout:grafana.smithi089 smithi089 *:3000 running (7m) 74s ago 16m 47.2M - dad864ee21e9 ec4d6738bce7 2024-09-21T04:19:48.245 INFO:teuthology.orchestra.run.smithi089.stdout:mds.foofs.smithi089.osufws smithi089 running (11m) 74s ago 11m 38.5M - 17.2.0 e1d6a67b021e 664d41859bea 2024-09-21T04:19:48.245 INFO:teuthology.orchestra.run.smithi089.stdout:mds.foofs.smithi195.hijxre smithi195 running (10m) 34s ago 10m 17.5M - 17.2.0 e1d6a67b021e 7bc0c6542d12 2024-09-21T04:19:48.245 INFO:teuthology.orchestra.run.smithi089.stdout:mgr.smithi089.srlyxf smithi089 *:8443,9283,8765 running (7m) 74s ago 18m 519M - 19.3.0-5103-gf36283e7 48ed0a2cf89d 3cbeaaa88241 2024-09-21T04:19:48.245 INFO:teuthology.orchestra.run.smithi089.stdout:mgr.smithi195.cgotzm smithi195 *:8443,9283,8765 running (6m) 34s ago 16m 457M - 19.3.0-5103-gf36283e7 48ed0a2cf89d d3734574cf66 2024-09-21T04:19:48.245 INFO:teuthology.orchestra.run.smithi089.stdout:mon.smithi089 smithi089 running (5m) 74s ago 18m 53.6M 2048M 19.3.0-5103-gf36283e7 48ed0a2cf89d 7dd34c8a6f21 2024-09-21T04:19:48.245 INFO:teuthology.orchestra.run.smithi089.stdout:mon.smithi195 smithi195 running (5m) 34s ago 16m 42.9M 2048M 19.3.0-5103-gf36283e7 48ed0a2cf89d 4255937444e7 2024-09-21T04:19:48.246 INFO:teuthology.orchestra.run.smithi089.stdout:nfs.foo.0.0.smithi089.dcjmqc smithi089 *:2049 running (10m) 74s ago 10m 290M - 4.0 e1d6a67b021e 98550f94e773 2024-09-21T04:19:48.246 INFO:teuthology.orchestra.run.smithi089.stdout:nfs.foo.1.0.smithi195.vhoayp smithi195 *:2049 running (10m) 34s ago 10m 88.5M - 4.0 e1d6a67b021e c60a4040130a 2024-09-21T04:19:48.246 INFO:teuthology.orchestra.run.smithi089.stdout:node-exporter.smithi089 smithi089 *:9100 running (7m) 74s ago 16m 9.84M - 1.7.0 72c9c2088986 7ef0d1588475 2024-09-21T04:19:48.246 INFO:teuthology.orchestra.run.smithi089.stdout:node-exporter.smithi195 smithi195 *:9100 running (7m) 34s ago 16m 10.4M - 1.7.0 72c9c2088986 61eafd61bc6c 2024-09-21T04:19:48.246 INFO:teuthology.orchestra.run.smithi089.stdout:osd.0 smithi089 running (3m) 74s ago 15m 331M 1935M 19.3.0-5103-gf36283e7 48ed0a2cf89d 3df0adea59fb 2024-09-21T04:19:48.246 INFO:teuthology.orchestra.run.smithi089.stdout:osd.1 smithi089 running (3m) 74s ago 15m 265M 1935M 19.3.0-5103-gf36283e7 48ed0a2cf89d da2e49774e7e 2024-09-21T04:19:48.246 INFO:teuthology.orchestra.run.smithi089.stdout:osd.2 smithi089 running (2m) 74s ago 14m 202M 1935M 19.3.0-5103-gf36283e7 48ed0a2cf89d b9f6438dd5b9 2024-09-21T04:19:48.246 INFO:teuthology.orchestra.run.smithi089.stdout:osd.3 smithi089 running (84s) 74s ago 14m 11.5M 1935M 19.3.0-5103-gf36283e7 48ed0a2cf89d d269cd94bd08 2024-09-21T04:19:48.246 INFO:teuthology.orchestra.run.smithi089.stdout:osd.4 smithi195 running (38s) 34s ago 13m 14.4M 2704M 19.3.0-5103-gf36283e7 48ed0a2cf89d e375f90f95c4 2024-09-21T04:19:48.246 INFO:teuthology.orchestra.run.smithi089.stdout:osd.5 smithi195 starting - - - 2704M 2024-09-21T04:19:48.246 INFO:teuthology.orchestra.run.smithi089.stdout:osd.6 smithi195 running (12m) 34s ago 12m 452M 2704M 17.2.0 e1d6a67b021e 0d7db362cc23 2024-09-21T04:19:48.246 INFO:teuthology.orchestra.run.smithi089.stdout:osd.7 smithi195 running (12m) 34s ago 12m 689M 2704M 17.2.0 e1d6a67b021e 0109b3fea188 2024-09-21T04:19:48.246 INFO:teuthology.orchestra.run.smithi089.stdout:prometheus.smithi089 smithi089 *:9095 running (6m) 74s ago 15m 53.3M - 2.51.0 1d3b7f56885b 8c9f309a6459 2024-09-21T04:19:48.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:19:48 smithi195 ceph-mon[49435]: from='client.34388 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T04:19:48.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:19:48 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:19:48.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:19:48 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T04:19:48.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:19:48 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:19:48.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:19:48 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:19:48.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:19:48 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T04:19:48.380 INFO:teuthology.orchestra.run.smithi089.stdout: 5 237532 9.18 MB/sec execute 507 sec latency 42.589 ms 2024-09-21T04:19:48.911 INFO:teuthology.orchestra.run.smithi089.stdout:{ 2024-09-21T04:19:48.911 INFO:teuthology.orchestra.run.smithi089.stdout: "mon": { 2024-09-21T04:19:48.911 INFO:teuthology.orchestra.run.smithi089.stdout: "ceph version 19.3.0-5103-gf36283e7 (f36283e7c91cac1c01cb9a7c4ff21bd697dbd4f2) squid (dev)": 2 2024-09-21T04:19:48.911 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2024-09-21T04:19:48.911 INFO:teuthology.orchestra.run.smithi089.stdout: "mgr": { 2024-09-21T04:19:48.911 INFO:teuthology.orchestra.run.smithi089.stdout: "ceph version 19.3.0-5103-gf36283e7 (f36283e7c91cac1c01cb9a7c4ff21bd697dbd4f2) squid (dev)": 2 2024-09-21T04:19:48.911 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2024-09-21T04:19:48.911 INFO:teuthology.orchestra.run.smithi089.stdout: "osd": { 2024-09-21T04:19:48.911 INFO:teuthology.orchestra.run.smithi089.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2, 2024-09-21T04:19:48.911 INFO:teuthology.orchestra.run.smithi089.stdout: "ceph version 19.3.0-5103-gf36283e7 (f36283e7c91cac1c01cb9a7c4ff21bd697dbd4f2) squid (dev)": 5 2024-09-21T04:19:48.912 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2024-09-21T04:19:48.912 INFO:teuthology.orchestra.run.smithi089.stdout: "mds": { 2024-09-21T04:19:48.912 INFO:teuthology.orchestra.run.smithi089.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-21T04:19:48.912 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2024-09-21T04:19:48.912 INFO:teuthology.orchestra.run.smithi089.stdout: "overall": { 2024-09-21T04:19:48.912 INFO:teuthology.orchestra.run.smithi089.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 4, 2024-09-21T04:19:48.912 INFO:teuthology.orchestra.run.smithi089.stdout: "ceph version 19.3.0-5103-gf36283e7 (f36283e7c91cac1c01cb9a7c4ff21bd697dbd4f2) squid (dev)": 9 2024-09-21T04:19:48.912 INFO:teuthology.orchestra.run.smithi089.stdout: } 2024-09-21T04:19:48.912 INFO:teuthology.orchestra.run.smithi089.stdout:} 2024-09-21T04:19:49.096 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:19:49 smithi089 ceph-mon[64076]: from='client.44271 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T04:19:49.097 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:19:49 smithi089 ceph-mon[64076]: pgmap v239: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 342 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 9.1 KiB/s rd, 4.8 MiB/s wr, 503 op/s; 181/1680 objects degraded (10.774%) 2024-09-21T04:19:49.097 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:19:49 smithi089 ceph-mon[64076]: Health check update: Degraded data redundancy: 181/1680 objects degraded (10.774%), 23 pgs degraded (PG_DEGRADED) 2024-09-21T04:19:49.097 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:19:49 smithi089 ceph-mon[64076]: from='client.? 172.21.15.89:0/453600186' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T04:19:49.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:19:49 smithi195 ceph-mon[49435]: from='client.44271 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T04:19:49.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:19:49 smithi195 ceph-mon[49435]: pgmap v239: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 342 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 9.1 KiB/s rd, 4.8 MiB/s wr, 503 op/s; 181/1680 objects degraded (10.774%) 2024-09-21T04:19:49.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:19:49 smithi195 ceph-mon[49435]: Health check update: Degraded data redundancy: 181/1680 objects degraded (10.774%), 23 pgs degraded (PG_DEGRADED) 2024-09-21T04:19:49.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:19:49 smithi195 ceph-mon[49435]: from='client.? 172.21.15.89:0/453600186' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T04:19:49.380 INFO:teuthology.orchestra.run.smithi089.stdout: 5 237877 9.19 MB/sec execute 508 sec latency 44.954 ms 2024-09-21T04:19:49.478 INFO:teuthology.orchestra.run.smithi089.stdout:{ 2024-09-21T04:19:49.478 INFO:teuthology.orchestra.run.smithi089.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:2195e5f178be0a9390728e2a14088236d8fe6f82635f5c84418fe37a04457e00", 2024-09-21T04:19:49.479 INFO:teuthology.orchestra.run.smithi089.stdout: "in_progress": true, 2024-09-21T04:19:49.479 INFO:teuthology.orchestra.run.smithi089.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-21T04:19:49.479 INFO:teuthology.orchestra.run.smithi089.stdout: "services_complete": [ 2024-09-21T04:19:49.479 INFO:teuthology.orchestra.run.smithi089.stdout: "mgr", 2024-09-21T04:19:49.479 INFO:teuthology.orchestra.run.smithi089.stdout: "crash", 2024-09-21T04:19:49.479 INFO:teuthology.orchestra.run.smithi089.stdout: "mon" 2024-09-21T04:19:49.479 INFO:teuthology.orchestra.run.smithi089.stdout: ], 2024-09-21T04:19:49.479 INFO:teuthology.orchestra.run.smithi089.stdout: "progress": "11/23 daemons upgraded", 2024-09-21T04:19:49.479 INFO:teuthology.orchestra.run.smithi089.stdout: "message": "Currently upgrading osd daemons", 2024-09-21T04:19:49.479 INFO:teuthology.orchestra.run.smithi089.stdout: "is_paused": false 2024-09-21T04:19:49.479 INFO:teuthology.orchestra.run.smithi089.stdout:} 2024-09-21T04:19:49.859 INFO:journalctl@ceph.osd.5.smithi195.stdout:Sep 21 04:19:49 smithi195 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-5[55460]: 2024-09-21T04:19:49.602+0000 7f7992516640 -1 Falling back to public interface 2024-09-21T04:19:50.170 INFO:teuthology.orchestra.run.smithi089.stdout:HEALTH_WARN 1 osds down; Degraded data redundancy: 177/1650 objects degraded (10.727%), 23 pgs degraded 2024-09-21T04:19:50.170 INFO:teuthology.orchestra.run.smithi089.stdout:[WRN] OSD_DOWN: 1 osds down 2024-09-21T04:19:50.170 INFO:teuthology.orchestra.run.smithi089.stdout: osd.5 (root=default,host=smithi195) is down 2024-09-21T04:19:50.170 INFO:teuthology.orchestra.run.smithi089.stdout:[WRN] PG_DEGRADED: Degraded data redundancy: 177/1650 objects degraded (10.727%), 23 pgs degraded 2024-09-21T04:19:50.170 INFO:teuthology.orchestra.run.smithi089.stdout: pg 2.2 is active+undersized+degraded, acting [1,6] 2024-09-21T04:19:50.170 INFO:teuthology.orchestra.run.smithi089.stdout: pg 2.3 is active+undersized+degraded, acting [2,7] 2024-09-21T04:19:50.170 INFO:teuthology.orchestra.run.smithi089.stdout: pg 2.b is active+undersized+degraded, acting [2,4] 2024-09-21T04:19:50.170 INFO:teuthology.orchestra.run.smithi089.stdout: pg 2.c is active+undersized+degraded, acting [2,0] 2024-09-21T04:19:50.170 INFO:teuthology.orchestra.run.smithi089.stdout: pg 2.12 is active+undersized+degraded, acting [3,7] 2024-09-21T04:19:50.170 INFO:teuthology.orchestra.run.smithi089.stdout: pg 2.14 is active+undersized+degraded, acting [6,3] 2024-09-21T04:19:50.170 INFO:teuthology.orchestra.run.smithi089.stdout: pg 2.15 is active+undersized+degraded, acting [1,0] 2024-09-21T04:19:50.171 INFO:teuthology.orchestra.run.smithi089.stdout: pg 2.16 is active+undersized+degraded, acting [6,2] 2024-09-21T04:19:50.171 INFO:teuthology.orchestra.run.smithi089.stdout: pg 2.17 is active+undersized+degraded, acting [6,2] 2024-09-21T04:19:50.171 INFO:teuthology.orchestra.run.smithi089.stdout: pg 2.18 is active+undersized+degraded, acting [4,7] 2024-09-21T04:19:50.171 INFO:teuthology.orchestra.run.smithi089.stdout: pg 2.1c is active+undersized+degraded, acting [7,2] 2024-09-21T04:19:50.171 INFO:teuthology.orchestra.run.smithi089.stdout: pg 2.1e is active+undersized+degraded, acting [3,0] 2024-09-21T04:19:50.171 INFO:teuthology.orchestra.run.smithi089.stdout: pg 3.2 is active+undersized+degraded, acting [3,6] 2024-09-21T04:19:50.171 INFO:teuthology.orchestra.run.smithi089.stdout: pg 3.4 is active+undersized+degraded, acting [1,2] 2024-09-21T04:19:50.171 INFO:teuthology.orchestra.run.smithi089.stdout: pg 3.5 is active+undersized+degraded, acting [3,2] 2024-09-21T04:19:50.171 INFO:teuthology.orchestra.run.smithi089.stdout: pg 3.c is active+undersized+degraded, acting [3,6] 2024-09-21T04:19:50.171 INFO:teuthology.orchestra.run.smithi089.stdout: pg 3.d is active+undersized+degraded, acting [7,6] 2024-09-21T04:19:50.171 INFO:teuthology.orchestra.run.smithi089.stdout: pg 3.10 is active+undersized+degraded, acting [6,0] 2024-09-21T04:19:50.171 INFO:teuthology.orchestra.run.smithi089.stdout: pg 3.16 is active+undersized+degraded, acting [7,1] 2024-09-21T04:19:50.171 INFO:teuthology.orchestra.run.smithi089.stdout: pg 3.17 is active+undersized+degraded, acting [0,3] 2024-09-21T04:19:50.171 INFO:teuthology.orchestra.run.smithi089.stdout: pg 3.1c is active+undersized+degraded, acting [4,1] 2024-09-21T04:19:50.172 INFO:teuthology.orchestra.run.smithi089.stdout: pg 3.1d is active+undersized+degraded, acting [4,6] 2024-09-21T04:19:50.172 INFO:teuthology.orchestra.run.smithi089.stdout: pg 3.1f is active+undersized+degraded, acting [0,2] 2024-09-21T04:19:50.281 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:19:50 smithi089 ceph-mon[64076]: from='client.34398 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T04:19:50.326 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:19:50 smithi195 ceph-mon[49435]: from='client.34398 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T04:19:50.380 INFO:teuthology.orchestra.run.smithi089.stdout: 5 238275 9.19 MB/sec execute 509 sec latency 39.351 ms 2024-09-21T04:19:51.306 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:19:51 smithi089 ceph-mon[64076]: from='client.34406 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T04:19:51.306 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:19:51 smithi089 ceph-mon[64076]: pgmap v240: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 337 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 7.8 KiB/s rd, 5.3 MiB/s wr, 534 op/s; 177/1650 objects degraded (10.727%) 2024-09-21T04:19:51.306 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:19:51 smithi089 ceph-mon[64076]: from='client.? 172.21.15.89:0/1638285259' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-21T04:19:51.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:19:51 smithi195 ceph-mon[49435]: from='client.34406 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T04:19:51.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:19:51 smithi195 ceph-mon[49435]: pgmap v240: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 337 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 7.8 KiB/s rd, 5.3 MiB/s wr, 534 op/s; 177/1650 objects degraded (10.727%) 2024-09-21T04:19:51.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:19:51 smithi195 ceph-mon[49435]: from='client.? 172.21.15.89:0/1638285259' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-21T04:19:51.388 INFO:teuthology.orchestra.run.smithi089.stdout: 5 238558 9.19 MB/sec execute 510 sec latency 46.873 ms 2024-09-21T04:19:52.388 INFO:teuthology.orchestra.run.smithi089.stdout: 5 238911 9.19 MB/sec execute 511 sec latency 46.153 ms 2024-09-21T04:19:53.073 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:19:53 smithi089 ceph-mon[64076]: pgmap v241: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 337 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 9.4 KiB/s rd, 4.7 MiB/s wr, 465 op/s; 177/1641 objects degraded (10.786%) 2024-09-21T04:19:53.073 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:19:53 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:19:53.073 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:19:53 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:19:53.073 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:19:53 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:19:53.073 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:19:53 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:19:53.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:19:53 smithi195 ceph-mon[49435]: pgmap v241: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 337 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 9.4 KiB/s rd, 4.7 MiB/s wr, 465 op/s; 177/1641 objects degraded (10.786%) 2024-09-21T04:19:53.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:19:53 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:19:53.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:19:53 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:19:53.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:19:53 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:19:53.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:19:53 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:19:53.388 INFO:teuthology.orchestra.run.smithi089.stdout: 5 239293 9.18 MB/sec execute 512 sec latency 41.983 ms 2024-09-21T04:19:54.345 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:19:54 smithi089 ceph-mon[64076]: Health check update: Degraded data redundancy: 181/1662 objects degraded (10.890%), 23 pgs degraded (PG_DEGRADED) 2024-09-21T04:19:54.388 INFO:teuthology.orchestra.run.smithi089.stdout: 5 239598 9.18 MB/sec execute 513 sec latency 37.297 ms 2024-09-21T04:19:54.554 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:19:54 smithi195 ceph-mon[49435]: Health check update: Degraded data redundancy: 181/1662 objects degraded (10.890%), 23 pgs degraded (PG_DEGRADED) 2024-09-21T04:19:55.362 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:19:55 smithi089 ceph-mon[64076]: pgmap v242: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 336 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.6 KiB/s rd, 5.9 MiB/s wr, 575 op/s; 181/1662 objects degraded (10.890%) 2024-09-21T04:19:55.388 INFO:teuthology.orchestra.run.smithi089.stdout: 5 239971 9.19 MB/sec execute 514 sec latency 51.336 ms 2024-09-21T04:19:55.513 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:19:55 smithi195 ceph-mon[49435]: pgmap v242: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 336 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.6 KiB/s rd, 5.9 MiB/s wr, 575 op/s; 181/1662 objects degraded (10.890%) 2024-09-21T04:19:56.359 INFO:journalctl@ceph.osd.5.smithi195.stdout:Sep 21 04:19:56 smithi195 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-5[55460]: 2024-09-21T04:19:56.071+0000 7f7992516640 -1 osd.5 0 read_superblock omap replica is missing. 2024-09-21T04:19:56.388 INFO:teuthology.orchestra.run.smithi089.stdout: 5 240379 9.18 MB/sec execute 515 sec latency 61.168 ms 2024-09-21T04:19:57.144 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:19:57 smithi089 ceph-mon[64076]: pgmap v243: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 335 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 9.7 KiB/s rd, 4.9 MiB/s wr, 523 op/s; 184/1686 objects degraded (10.913%) 2024-09-21T04:19:57.388 INFO:teuthology.orchestra.run.smithi089.stdout: 5 240762 9.18 MB/sec execute 516 sec latency 33.391 ms 2024-09-21T04:19:57.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:19:57 smithi195 ceph-mon[49435]: pgmap v243: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 335 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 9.7 KiB/s rd, 4.9 MiB/s wr, 523 op/s; 184/1686 objects degraded (10.913%) 2024-09-21T04:19:57.609 INFO:journalctl@ceph.osd.5.smithi195.stdout:Sep 21 04:19:57 smithi195 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-5[55460]: 2024-09-21T04:19:57.297+0000 7f7992516640 -1 osd.5 135 log_to_monitors true 2024-09-21T04:19:58.173 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:19:58 smithi089 ceph-mon[64076]: from='osd.5 [v2:172.21.15.195:6808/3875427982,v1:172.21.15.195:6809/3875427982]' entity='osd.5' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]} : dispatch 2024-09-21T04:19:58.173 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:19:58 smithi089 ceph-mon[64076]: from='osd.5 ' entity='osd.5' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]} : dispatch 2024-09-21T04:19:58.173 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:19:58 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:19:58.173 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:19:58 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:19:58.173 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:19:58 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T04:19:58.173 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:19:58 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T04:19:58.173 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:19:58 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:19:58.173 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:19:58 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:19:58.388 INFO:teuthology.orchestra.run.smithi089.stdout: 5 241173 9.20 MB/sec execute 517 sec latency 30.788 ms 2024-09-21T04:19:58.609 INFO:journalctl@ceph.osd.5.smithi195.stdout:Sep 21 04:19:58 smithi195 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-5[55460]: 2024-09-21T04:19:58.150+0000 7f798a2c0640 -1 osd.5 135 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-21T04:19:58.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:19:58 smithi195 ceph-mon[49435]: from='osd.5 [v2:172.21.15.195:6808/3875427982,v1:172.21.15.195:6809/3875427982]' entity='osd.5' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]} : dispatch 2024-09-21T04:19:58.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:19:58 smithi195 ceph-mon[49435]: from='osd.5 ' entity='osd.5' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]} : dispatch 2024-09-21T04:19:58.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:19:58 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:19:58.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:19:58 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:19:58.610 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:19:58 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T04:19:58.610 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:19:58 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T04:19:58.610 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:19:58 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:19:58.610 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:19:58 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:19:59.195 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:19:59 smithi089 ceph-mon[64076]: pgmap v244: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 334 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 8.1 KiB/s rd, 5.3 MiB/s wr, 501 op/s; 180/1689 objects degraded (10.657%) 2024-09-21T04:19:59.195 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:19:59 smithi089 ceph-mon[64076]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-09-21T04:19:59.195 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:19:59 smithi089 ceph-mon[64076]: from='osd.5 [v2:172.21.15.195:6808/3875427982,v1:172.21.15.195:6809/3875427982]' entity='osd.5' cmd={"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi195", "root=default"]} : dispatch 2024-09-21T04:19:59.195 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:19:59 smithi089 ceph-mon[64076]: osdmap e138: 8 total, 7 up, 8 in 2024-09-21T04:19:59.195 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:19:59 smithi089 ceph-mon[64076]: from='osd.5 ' entity='osd.5' cmd={"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi195", "root=default"]} : dispatch 2024-09-21T04:19:59.195 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:19:59 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T04:19:59.195 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:19:59 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "versions"} : dispatch 2024-09-21T04:19:59.195 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:19:59 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "versions"} : dispatch 2024-09-21T04:19:59.195 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:19:59 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "versions"} : dispatch 2024-09-21T04:19:59.195 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:19:59 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16} : dispatch 2024-09-21T04:19:59.388 INFO:teuthology.orchestra.run.smithi089.stdout: 5 241536 9.20 MB/sec execute 518 sec latency 235.873 ms 2024-09-21T04:19:59.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:19:59 smithi195 ceph-mon[49435]: pgmap v244: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 334 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 8.1 KiB/s rd, 5.3 MiB/s wr, 501 op/s; 180/1689 objects degraded (10.657%) 2024-09-21T04:19:59.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:19:59 smithi195 ceph-mon[49435]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-09-21T04:19:59.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:19:59 smithi195 ceph-mon[49435]: from='osd.5 [v2:172.21.15.195:6808/3875427982,v1:172.21.15.195:6809/3875427982]' entity='osd.5' cmd={"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi195", "root=default"]} : dispatch 2024-09-21T04:19:59.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:19:59 smithi195 ceph-mon[49435]: osdmap e138: 8 total, 7 up, 8 in 2024-09-21T04:19:59.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:19:59 smithi195 ceph-mon[49435]: from='osd.5 ' entity='osd.5' cmd={"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi195", "root=default"]} : dispatch 2024-09-21T04:19:59.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:19:59 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T04:19:59.610 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:19:59 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "versions"} : dispatch 2024-09-21T04:19:59.610 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:19:59 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "versions"} : dispatch 2024-09-21T04:19:59.610 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:19:59 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "versions"} : dispatch 2024-09-21T04:19:59.610 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:19:59 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16} : dispatch 2024-09-21T04:20:00.388 INFO:teuthology.orchestra.run.smithi089.stdout: 5 241542 9.18 MB/sec execute 519 sec latency 1235.928 ms 2024-09-21T04:20:00.515 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:20:00 smithi089 ceph-mon[64076]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-09-21T04:20:00.515 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:20:00 smithi089 ceph-mon[64076]: Upgrade: unsafe to stop osd(s) at this time (13 PGs are or would become offline) 2024-09-21T04:20:00.515 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:20:00 smithi089 ceph-mon[64076]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-21T04:20:00.516 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:20:00 smithi089 ceph-mon[64076]: osd.5 [v2:172.21.15.195:6808/3875427982,v1:172.21.15.195:6809/3875427982] boot 2024-09-21T04:20:00.516 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:20:00 smithi089 ceph-mon[64076]: osdmap e139: 8 total, 8 up, 8 in 2024-09-21T04:20:00.516 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:20:00 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "osd metadata", "id": 5} : dispatch 2024-09-21T04:20:00.516 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:20:00 smithi089 ceph-mon[64076]: Health check update: Degraded data redundancy: 180/1689 objects degraded (10.657%), 23 pgs degraded (PG_DEGRADED) 2024-09-21T04:20:00.516 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:20:00 smithi089 ceph-mon[64076]: osdmap e140: 8 total, 8 up, 8 in 2024-09-21T04:20:00.516 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:20:00 smithi089 ceph-mon[64076]: Health detail: HEALTH_WARN Degraded data redundancy: 180/1689 objects degraded (10.657%), 23 pgs degraded 2024-09-21T04:20:00.517 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:20:00 smithi089 ceph-mon[64076]: [WRN] PG_DEGRADED: Degraded data redundancy: 180/1689 objects degraded (10.657%), 23 pgs degraded 2024-09-21T04:20:00.517 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:20:00 smithi089 ceph-mon[64076]: pg 2.2 is active+undersized+degraded, acting [1,6] 2024-09-21T04:20:00.517 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:20:00 smithi089 ceph-mon[64076]: pg 2.3 is active+undersized+degraded, acting [2,7] 2024-09-21T04:20:00.517 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:20:00 smithi089 ceph-mon[64076]: pg 2.b is active+undersized+degraded, acting [2,4] 2024-09-21T04:20:00.517 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:20:00 smithi089 ceph-mon[64076]: pg 2.c is active+undersized+degraded, acting [2,0] 2024-09-21T04:20:00.517 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:20:00 smithi089 ceph-mon[64076]: pg 2.12 is active+undersized+degraded, acting [3,7] 2024-09-21T04:20:00.518 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:20:00 smithi089 ceph-mon[64076]: pg 2.14 is active+undersized+degraded, acting [6,3] 2024-09-21T04:20:00.518 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:20:00 smithi089 ceph-mon[64076]: pg 2.15 is active+undersized+degraded, acting [1,0] 2024-09-21T04:20:00.518 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:20:00 smithi089 ceph-mon[64076]: pg 2.16 is active+undersized+degraded, acting [6,2] 2024-09-21T04:20:00.518 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:20:00 smithi089 ceph-mon[64076]: pg 2.17 is active+undersized+degraded, acting [6,2] 2024-09-21T04:20:00.518 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:20:00 smithi089 ceph-mon[64076]: pg 2.18 is active+undersized+degraded, acting [4,7] 2024-09-21T04:20:00.518 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:20:00 smithi089 ceph-mon[64076]: pg 2.1c is active+undersized+degraded, acting [7,2] 2024-09-21T04:20:00.519 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:20:00 smithi089 ceph-mon[64076]: pg 2.1e is active+undersized+degraded, acting [3,0] 2024-09-21T04:20:00.519 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:20:00 smithi089 ceph-mon[64076]: pg 3.2 is active+undersized+degraded, acting [3,6] 2024-09-21T04:20:00.519 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:20:00 smithi089 ceph-mon[64076]: pg 3.4 is active+undersized+degraded, acting [1,2] 2024-09-21T04:20:00.519 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:20:00 smithi089 ceph-mon[64076]: pg 3.5 is active+undersized+degraded, acting [3,2] 2024-09-21T04:20:00.519 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:20:00 smithi089 ceph-mon[64076]: pg 3.c is active+undersized+degraded, acting [3,6] 2024-09-21T04:20:00.519 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:20:00 smithi089 ceph-mon[64076]: pg 3.d is active+undersized+degraded, acting [7,6] 2024-09-21T04:20:00.519 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:20:00 smithi089 ceph-mon[64076]: pg 3.10 is active+undersized+degraded, acting [6,0] 2024-09-21T04:20:00.520 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:20:00 smithi089 ceph-mon[64076]: pg 3.16 is active+undersized+degraded, acting [7,1] 2024-09-21T04:20:00.520 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:20:00 smithi089 ceph-mon[64076]: pg 3.17 is active+undersized+degraded, acting [0,3] 2024-09-21T04:20:00.520 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:20:00 smithi089 ceph-mon[64076]: pg 3.1c is active+undersized+degraded, acting [4,1] 2024-09-21T04:20:00.520 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:20:00 smithi089 ceph-mon[64076]: pg 3.1d is active+undersized+degraded, acting [4,6] 2024-09-21T04:20:00.520 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:20:00 smithi089 ceph-mon[64076]: pg 3.1f is active+undersized+degraded, acting [0,2] 2024-09-21T04:20:00.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:20:00 smithi195 ceph-mon[49435]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-09-21T04:20:00.610 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:20:00 smithi195 ceph-mon[49435]: Upgrade: unsafe to stop osd(s) at this time (13 PGs are or would become offline) 2024-09-21T04:20:00.610 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:20:00 smithi195 ceph-mon[49435]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-21T04:20:00.610 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:20:00 smithi195 ceph-mon[49435]: osd.5 [v2:172.21.15.195:6808/3875427982,v1:172.21.15.195:6809/3875427982] boot 2024-09-21T04:20:00.610 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:20:00 smithi195 ceph-mon[49435]: osdmap e139: 8 total, 8 up, 8 in 2024-09-21T04:20:00.610 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:20:00 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "osd metadata", "id": 5} : dispatch 2024-09-21T04:20:00.610 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:20:00 smithi195 ceph-mon[49435]: Health check update: Degraded data redundancy: 180/1689 objects degraded (10.657%), 23 pgs degraded (PG_DEGRADED) 2024-09-21T04:20:00.610 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:20:00 smithi195 ceph-mon[49435]: osdmap e140: 8 total, 8 up, 8 in 2024-09-21T04:20:00.610 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:20:00 smithi195 ceph-mon[49435]: Health detail: HEALTH_WARN Degraded data redundancy: 180/1689 objects degraded (10.657%), 23 pgs degraded 2024-09-21T04:20:00.610 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:20:00 smithi195 ceph-mon[49435]: [WRN] PG_DEGRADED: Degraded data redundancy: 180/1689 objects degraded (10.657%), 23 pgs degraded 2024-09-21T04:20:00.610 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:20:00 smithi195 ceph-mon[49435]: pg 2.2 is active+undersized+degraded, acting [1,6] 2024-09-21T04:20:00.610 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:20:00 smithi195 ceph-mon[49435]: pg 2.3 is active+undersized+degraded, acting [2,7] 2024-09-21T04:20:00.610 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:20:00 smithi195 ceph-mon[49435]: pg 2.b is active+undersized+degraded, acting [2,4] 2024-09-21T04:20:00.610 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:20:00 smithi195 ceph-mon[49435]: pg 2.c is active+undersized+degraded, acting [2,0] 2024-09-21T04:20:00.610 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:20:00 smithi195 ceph-mon[49435]: pg 2.12 is active+undersized+degraded, acting [3,7] 2024-09-21T04:20:00.611 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:20:00 smithi195 ceph-mon[49435]: pg 2.14 is active+undersized+degraded, acting [6,3] 2024-09-21T04:20:00.611 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:20:00 smithi195 ceph-mon[49435]: pg 2.15 is active+undersized+degraded, acting [1,0] 2024-09-21T04:20:00.611 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:20:00 smithi195 ceph-mon[49435]: pg 2.16 is active+undersized+degraded, acting [6,2] 2024-09-21T04:20:00.611 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:20:00 smithi195 ceph-mon[49435]: pg 2.17 is active+undersized+degraded, acting [6,2] 2024-09-21T04:20:00.611 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:20:00 smithi195 ceph-mon[49435]: pg 2.18 is active+undersized+degraded, acting [4,7] 2024-09-21T04:20:00.611 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:20:00 smithi195 ceph-mon[49435]: pg 2.1c is active+undersized+degraded, acting [7,2] 2024-09-21T04:20:00.611 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:20:00 smithi195 ceph-mon[49435]: pg 2.1e is active+undersized+degraded, acting [3,0] 2024-09-21T04:20:00.611 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:20:00 smithi195 ceph-mon[49435]: pg 3.2 is active+undersized+degraded, acting [3,6] 2024-09-21T04:20:00.611 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:20:00 smithi195 ceph-mon[49435]: pg 3.4 is active+undersized+degraded, acting [1,2] 2024-09-21T04:20:00.611 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:20:00 smithi195 ceph-mon[49435]: pg 3.5 is active+undersized+degraded, acting [3,2] 2024-09-21T04:20:00.611 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:20:00 smithi195 ceph-mon[49435]: pg 3.c is active+undersized+degraded, acting [3,6] 2024-09-21T04:20:00.611 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:20:00 smithi195 ceph-mon[49435]: pg 3.d is active+undersized+degraded, acting [7,6] 2024-09-21T04:20:00.611 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:20:00 smithi195 ceph-mon[49435]: pg 3.10 is active+undersized+degraded, acting [6,0] 2024-09-21T04:20:00.611 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:20:00 smithi195 ceph-mon[49435]: pg 3.16 is active+undersized+degraded, acting [7,1] 2024-09-21T04:20:00.612 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:20:00 smithi195 ceph-mon[49435]: pg 3.17 is active+undersized+degraded, acting [0,3] 2024-09-21T04:20:00.612 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:20:00 smithi195 ceph-mon[49435]: pg 3.1c is active+undersized+degraded, acting [4,1] 2024-09-21T04:20:00.612 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:20:00 smithi195 ceph-mon[49435]: pg 3.1d is active+undersized+degraded, acting [4,6] 2024-09-21T04:20:00.612 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:20:00 smithi195 ceph-mon[49435]: pg 3.1f is active+undersized+degraded, acting [0,2] 2024-09-21T04:20:01.349 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:20:01 smithi089 ceph-mon[64076]: pgmap v248: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 339 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 5.5 MiB/s wr, 569 op/s; 176/1665 objects degraded (10.571%) 2024-09-21T04:20:01.349 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:20:01 smithi089 ceph-mon[64076]: osdmap e141: 8 total, 8 up, 8 in 2024-09-21T04:20:01.388 INFO:teuthology.orchestra.run.smithi089.stdout: 5 241662 9.17 MB/sec execute 520 sec latency 1919.090 ms 2024-09-21T04:20:01.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:20:01 smithi195 ceph-mon[49435]: pgmap v248: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 339 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 5.5 MiB/s wr, 569 op/s; 176/1665 objects degraded (10.571%) 2024-09-21T04:20:01.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:20:01 smithi195 ceph-mon[49435]: osdmap e141: 8 total, 8 up, 8 in 2024-09-21T04:20:02.388 INFO:teuthology.orchestra.run.smithi089.stdout: 5 241945 9.17 MB/sec execute 521 sec latency 457.006 ms 2024-09-21T04:20:03.183 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:20:02 smithi089 ceph-mon[64076]: osdmap e142: 8 total, 8 up, 8 in 2024-09-21T04:20:03.183 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:20:02 smithi089 ceph-mon[64076]: pgmap v251: 97 pgs: 1 remapped+peering, 8 active+undersized, 20 active+undersized+degraded, 68 active+clean; 331 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 4.0 MiB/s wr, 458 op/s; 157/1653 objects degraded (9.498%); 0 B/s, 0 objects/s recovering 2024-09-21T04:20:03.183 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:20:02 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:20:03.183 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:20:02 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T04:20:03.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:20:02 smithi195 ceph-mon[49435]: osdmap e142: 8 total, 8 up, 8 in 2024-09-21T04:20:03.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:20:02 smithi195 ceph-mon[49435]: pgmap v251: 97 pgs: 1 remapped+peering, 8 active+undersized, 20 active+undersized+degraded, 68 active+clean; 331 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 4.0 MiB/s wr, 458 op/s; 157/1653 objects degraded (9.498%); 0 B/s, 0 objects/s recovering 2024-09-21T04:20:03.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:20:02 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:20:03.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:20:02 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T04:20:03.388 INFO:teuthology.orchestra.run.smithi089.stdout: 5 242063 9.16 MB/sec execute 522 sec latency 1024.737 ms 2024-09-21T04:20:03.945 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:20:03 smithi089 ceph-mon[64076]: osdmap e143: 8 total, 8 up, 8 in 2024-09-21T04:20:04.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:20:03 smithi195 ceph-mon[49435]: osdmap e143: 8 total, 8 up, 8 in 2024-09-21T04:20:04.388 INFO:teuthology.orchestra.run.smithi089.stdout: 5 242358 9.16 MB/sec execute 523 sec latency 32.225 ms 2024-09-21T04:20:04.966 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:20:04 smithi089 ceph-mon[64076]: pgmap v253: 97 pgs: 1 peering, 2 active+undersized, 94 active+clean; 329 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 4.6 KiB/s rd, 3.3 MiB/s wr, 325 op/s; 2.9 MiB/s, 67 objects/s recovering 2024-09-21T04:20:04.966 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:20:04 smithi089 ceph-mon[64076]: Health check update: Degraded data redundancy: 157/1653 objects degraded (9.498%), 20 pgs degraded (PG_DEGRADED) 2024-09-21T04:20:05.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:20:04 smithi195 ceph-mon[49435]: pgmap v253: 97 pgs: 1 peering, 2 active+undersized, 94 active+clean; 329 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 4.6 KiB/s rd, 3.3 MiB/s wr, 325 op/s; 2.9 MiB/s, 67 objects/s recovering 2024-09-21T04:20:05.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:20:04 smithi195 ceph-mon[49435]: Health check update: Degraded data redundancy: 157/1653 objects degraded (9.498%), 20 pgs degraded (PG_DEGRADED) 2024-09-21T04:20:05.388 INFO:teuthology.orchestra.run.smithi089.stdout: 5 242736 9.15 MB/sec execute 524 sec latency 47.729 ms 2024-09-21T04:20:06.002 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:20:05 smithi089 ceph-mon[64076]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 157/1653 objects degraded (9.498%), 20 pgs degraded) 2024-09-21T04:20:06.002 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:20:05 smithi089 ceph-mon[64076]: Cluster is now healthy 2024-09-21T04:20:06.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:20:05 smithi195 ceph-mon[49435]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 157/1653 objects degraded (9.498%), 20 pgs degraded) 2024-09-21T04:20:06.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:20:05 smithi195 ceph-mon[49435]: Cluster is now healthy 2024-09-21T04:20:06.388 INFO:teuthology.orchestra.run.smithi089.stdout: 5 243052 9.15 MB/sec execute 525 sec latency 30.516 ms 2024-09-21T04:20:07.035 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:20:06 smithi089 ceph-mon[64076]: pgmap v254: 97 pgs: 1 peering, 96 active+clean; 333 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 3.2 KiB/s rd, 3.1 MiB/s wr, 366 op/s; 2.0 MiB/s, 45 objects/s recovering 2024-09-21T04:20:07.260 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:20:06 smithi195 ceph-mon[49435]: pgmap v254: 97 pgs: 1 peering, 96 active+clean; 333 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 3.2 KiB/s rd, 3.1 MiB/s wr, 366 op/s; 2.0 MiB/s, 45 objects/s recovering 2024-09-21T04:20:07.388 INFO:teuthology.orchestra.run.smithi089.stdout: 5 243326 9.15 MB/sec execute 526 sec latency 39.312 ms 2024-09-21T04:20:08.388 INFO:teuthology.orchestra.run.smithi089.stdout: 5 243619 9.15 MB/sec execute 527 sec latency 112.838 ms 2024-09-21T04:20:09.317 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:20:09 smithi089 ceph-mon[64076]: pgmap v255: 97 pgs: 97 active+clean; 331 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 7.9 KiB/s rd, 3.6 MiB/s wr, 359 op/s; 1.7 MiB/s, 38 objects/s recovering 2024-09-21T04:20:09.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:20:09 smithi195 ceph-mon[49435]: pgmap v255: 97 pgs: 97 active+clean; 331 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 7.9 KiB/s rd, 3.6 MiB/s wr, 359 op/s; 1.7 MiB/s, 38 objects/s recovering 2024-09-21T04:20:09.388 INFO:teuthology.orchestra.run.smithi089.stdout: 5 244060 9.14 MB/sec execute 528 sec latency 62.275 ms 2024-09-21T04:20:10.388 INFO:teuthology.orchestra.run.smithi089.stdout: 5 244323 9.14 MB/sec execute 529 sec latency 37.769 ms 2024-09-21T04:20:11.114 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:20:11 smithi089 ceph-mon[64076]: pgmap v256: 97 pgs: 97 active+clean; 327 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 6.9 KiB/s rd, 4.0 MiB/s wr, 369 op/s; 1.5 MiB/s, 33 objects/s recovering 2024-09-21T04:20:11.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:20:11 smithi195 ceph-mon[49435]: pgmap v256: 97 pgs: 97 active+clean; 327 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 6.9 KiB/s rd, 4.0 MiB/s wr, 369 op/s; 1.5 MiB/s, 33 objects/s recovering 2024-09-21T04:20:11.388 INFO:teuthology.orchestra.run.smithi089.stdout: 5 244493 9.13 MB/sec execute 530 sec latency 48.175 ms 2024-09-21T04:20:12.389 INFO:teuthology.orchestra.run.smithi089.stdout: 5 244777 9.14 MB/sec execute 531 sec latency 70.299 ms 2024-09-21T04:20:13.179 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:20:13 smithi089 ceph-mon[64076]: pgmap v257: 97 pgs: 97 active+clean; 331 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 5.6 KiB/s rd, 3.9 MiB/s wr, 439 op/s; 1.2 MiB/s, 27 objects/s recovering 2024-09-21T04:20:13.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:20:13 smithi195 ceph-mon[49435]: pgmap v257: 97 pgs: 97 active+clean; 331 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 5.6 KiB/s rd, 3.9 MiB/s wr, 439 op/s; 1.2 MiB/s, 27 objects/s recovering 2024-09-21T04:20:13.388 INFO:teuthology.orchestra.run.smithi089.stdout: 5 245086 9.14 MB/sec execute 532 sec latency 62.437 ms 2024-09-21T04:20:14.199 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:20:14 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16} : dispatch 2024-09-21T04:20:14.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:20:14 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16} : dispatch 2024-09-21T04:20:14.388 INFO:teuthology.orchestra.run.smithi089.stdout: 5 245420 9.14 MB/sec execute 533 sec latency 56.132 ms 2024-09-21T04:20:15.212 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:20:15 smithi089 ceph-mon[64076]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-09-21T04:20:15.212 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:20:15 smithi089 ceph-mon[64076]: Upgrade: osd.6 is safe to restart 2024-09-21T04:20:15.213 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:20:15 smithi089 ceph-mon[64076]: pgmap v258: 97 pgs: 97 active+clean; 334 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 7.6 KiB/s rd, 4.9 MiB/s wr, 516 op/s; 1022 KiB/s, 21 objects/s recovering 2024-09-21T04:20:15.213 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:20:15 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:20:15.213 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:20:15 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "auth get", "entity": "osd.6"} : dispatch 2024-09-21T04:20:15.213 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:20:15 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T04:20:15.344 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:20:15 smithi195 ceph-mon[49435]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-09-21T04:20:15.344 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:20:15 smithi195 ceph-mon[49435]: Upgrade: osd.6 is safe to restart 2024-09-21T04:20:15.345 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:20:15 smithi195 ceph-mon[49435]: pgmap v258: 97 pgs: 97 active+clean; 334 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 7.6 KiB/s rd, 4.9 MiB/s wr, 516 op/s; 1022 KiB/s, 21 objects/s recovering 2024-09-21T04:20:15.345 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:20:15 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:20:15.345 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:20:15 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "auth get", "entity": "osd.6"} : dispatch 2024-09-21T04:20:15.345 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:20:15 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T04:20:15.388 INFO:teuthology.orchestra.run.smithi089.stdout: 5 245667 9.14 MB/sec execute 534 sec latency 37.980 ms 2024-09-21T04:20:16.225 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:20:16 smithi089 ceph-mon[64076]: Upgrade: Updating osd.6 2024-09-21T04:20:16.225 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:20:16 smithi089 ceph-mon[64076]: Deploying daemon osd.6 on smithi195 2024-09-21T04:20:16.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:20:16 smithi195 ceph-mon[49435]: Upgrade: Updating osd.6 2024-09-21T04:20:16.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:20:16 smithi195 ceph-mon[49435]: Deploying daemon osd.6 on smithi195 2024-09-21T04:20:16.388 INFO:teuthology.orchestra.run.smithi089.stdout: 5 245937 9.14 MB/sec execute 535 sec latency 44.015 ms 2024-09-21T04:20:17.280 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:20:17 smithi089 ceph-mon[64076]: pgmap v259: 97 pgs: 97 active+clean; 335 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 5.4 KiB/s rd, 4.0 MiB/s wr, 413 op/s 2024-09-21T04:20:17.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:20:17 smithi195 ceph-mon[49435]: pgmap v259: 97 pgs: 97 active+clean; 335 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 5.4 KiB/s rd, 4.0 MiB/s wr, 413 op/s 2024-09-21T04:20:17.388 INFO:teuthology.orchestra.run.smithi089.stdout: 5 246277 9.13 MB/sec execute 536 sec latency 54.223 ms 2024-09-21T04:20:18.389 INFO:teuthology.orchestra.run.smithi089.stdout: 5 246549 9.12 MB/sec execute 537 sec latency 47.693 ms 2024-09-21T04:20:18.560 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:20:18 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:20:18.560 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:20:18 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T04:20:18.560 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:20:18 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:20:18.560 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:20:18 smithi089 ceph-mon[64076]: pgmap v260: 97 pgs: 97 active+clean; 334 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 8.2 KiB/s rd, 4.3 MiB/s wr, 406 op/s 2024-09-21T04:20:18.609 INFO:journalctl@ceph.osd.6.smithi195.stdout:Sep 21 04:20:18 smithi195 systemd[1]: Stopping Ceph osd.6 for 1b9b79fc-77ce-11ef-baf5-efdc52797490... 2024-09-21T04:20:18.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:20:18 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:20:18.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:20:18 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T04:20:18.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:20:18 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:20:18.610 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:20:18 smithi195 ceph-mon[49435]: pgmap v260: 97 pgs: 97 active+clean; 334 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 8.2 KiB/s rd, 4.3 MiB/s wr, 406 op/s 2024-09-21T04:20:19.109 INFO:journalctl@ceph.osd.6.smithi195.stdout:Sep 21 04:20:18 smithi195 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-6[35964]: 2024-09-21T04:20:18.647+0000 7fde5ba0d700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.6 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-21T04:20:19.109 INFO:journalctl@ceph.osd.6.smithi195.stdout:Sep 21 04:20:18 smithi195 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-6[35964]: 2024-09-21T04:20:18.647+0000 7fde5ba0d700 -1 osd.6 143 *** Got signal Terminated *** 2024-09-21T04:20:19.109 INFO:journalctl@ceph.osd.6.smithi195.stdout:Sep 21 04:20:18 smithi195 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-6[35964]: 2024-09-21T04:20:18.647+0000 7fde5ba0d700 -1 osd.6 143 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-21T04:20:19.389 INFO:teuthology.orchestra.run.smithi089.stdout: 5 246702 9.11 MB/sec execute 538 sec latency 63.723 ms 2024-09-21T04:20:19.514 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:20:19 smithi089 ceph-mon[64076]: osd.6 marked itself down and dead 2024-09-21T04:20:19.859 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:20:19 smithi195 ceph-mon[49435]: osd.6 marked itself down and dead 2024-09-21T04:20:20.359 INFO:journalctl@ceph.osd.6.smithi195.stdout:Sep 21 04:20:20 smithi195 podman[57028]: 2024-09-21 04:20:20.103246655 +0000 UTC m=+1.582741943 container died 0d7db362cc2354099abb4405a6d116557c7117c71fdd163703054b0c6ded664f (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-6, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.component=centos-stream-container, io.openshift.expose-services=, vendor=Red Hat, Inc., com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, distribution-scope=public, RELEASE=HEAD, ceph=True, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.openshift.tags=base centos centos-stream, vcs-type=git, maintainer=Guillaume Abrioux , GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, architecture=x86_64, version=8, CEPH_POINT_RELEASE=-17.2.0, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_CLEAN=True, io.k8s.display-name=CentOS Stream 8, name=centos-stream, io.buildah.version=1.19.8, GIT_REPO=https://github.com/ceph/ceph-container.git, build-date=2022-05-03T08:36:31.336870, release=754, GIT_BRANCH=HEAD) 2024-09-21T04:20:20.389 INFO:teuthology.orchestra.run.smithi089.stdout: 5 246714 9.10 MB/sec execute 539 sec latency 939.705 ms 2024-09-21T04:20:20.595 INFO:teuthology.orchestra.run.smithi089.stdout:true 2024-09-21T04:20:20.706 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:20:20 smithi089 ceph-mon[64076]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-21T04:20:20.706 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:20:20 smithi089 ceph-mon[64076]: osdmap e144: 8 total, 7 up, 8 in 2024-09-21T04:20:20.706 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:20:20 smithi089 ceph-mon[64076]: pgmap v262: 97 pgs: 10 stale+active+clean, 87 active+clean; 334 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.4 MiB/s wr, 445 op/s 2024-09-21T04:20:20.859 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:20:20 smithi195 ceph-mon[49435]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-21T04:20:20.859 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:20:20 smithi195 ceph-mon[49435]: osdmap e144: 8 total, 7 up, 8 in 2024-09-21T04:20:20.859 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:20:20 smithi195 ceph-mon[49435]: pgmap v262: 97 pgs: 10 stale+active+clean, 87 active+clean; 334 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.4 MiB/s wr, 445 op/s 2024-09-21T04:20:20.860 INFO:journalctl@ceph.osd.6.smithi195.stdout:Sep 21 04:20:20 smithi195 podman[57028]: 2024-09-21 04:20:20.438508184 +0000 UTC m=+1.918003473 container cleanup 0d7db362cc2354099abb4405a6d116557c7117c71fdd163703054b0c6ded664f (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-6, io.openshift.expose-services=, com.redhat.component=centos-stream-container, ceph=True, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.buildah.version=1.19.8, GIT_CLEAN=True, GIT_BRANCH=HEAD, build-date=2022-05-03T08:36:31.336870, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., architecture=x86_64, maintainer=Guillaume Abrioux , com.redhat.license_terms=https://centos.org/legal/licensing-policy/, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, vcs-type=git, name=centos-stream, io.openshift.tags=base centos centos-stream, CEPH_POINT_RELEASE=-17.2.0, RELEASE=HEAD, version=8, GIT_REPO=https://github.com/ceph/ceph-container.git, release=754, io.k8s.display-name=CentOS Stream 8, distribution-scope=public, vendor=Red Hat, Inc., summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image) 2024-09-21T04:20:20.860 INFO:journalctl@ceph.osd.6.smithi195.stdout:Sep 21 04:20:20 smithi195 bash[57028]: ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-6 2024-09-21T04:20:20.860 INFO:journalctl@ceph.osd.6.smithi195.stdout:Sep 21 04:20:20 smithi195 podman[57045]: 2024-09-21 04:20:20.658288548 +0000 UTC m=+0.557606026 container remove 0d7db362cc2354099abb4405a6d116557c7117c71fdd163703054b0c6ded664f (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-6, RELEASE=HEAD, io.k8s.display-name=CentOS Stream 8, distribution-scope=public, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.license_terms=https://centos.org/legal/licensing-policy/, name=centos-stream, architecture=x86_64, version=8, io.openshift.tags=base centos centos-stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.component=centos-stream-container, vcs-type=git, GIT_CLEAN=True, GIT_BRANCH=HEAD, io.buildah.version=1.19.8, io.openshift.expose-services=, maintainer=Guillaume Abrioux , ceph=True, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_REPO=https://github.com/ceph/ceph-container.git, build-date=2022-05-03T08:36:31.336870, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, release=754, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vendor=Red Hat, Inc., com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, CEPH_POINT_RELEASE=-17.2.0) 2024-09-21T04:20:21.359 INFO:journalctl@ceph.osd.6.smithi195.stdout:Sep 21 04:20:21 smithi195 podman[57135]: 2024-09-21 04:20:21.001873368 +0000 UTC m=+0.016807799 image pull 48ed0a2cf89d6d0371a2c6185a5869a747265f6abd5ac2a74f00cc32d5628721 quay.ceph.io/ceph-ci/ceph@sha256:2195e5f178be0a9390728e2a14088236d8fe6f82635f5c84418fe37a04457e00 2024-09-21T04:20:21.359 INFO:journalctl@ceph.osd.6.smithi195.stdout:Sep 21 04:20:21 smithi195 podman[57135]: 2024-09-21 04:20:21.285574147 +0000 UTC m=+0.300508549 container create e2ee93a3976024e5f5fb91e5ce20237fb29c7f411afea2e642745b6662b927c2 (image=quay.ceph.io/ceph-ci/ceph@sha256:2195e5f178be0a9390728e2a14088236d8fe6f82635f5c84418fe37a04457e00, name=ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-6-deactivate, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, ceph=True, RELEASE=wip-guits-main-2024-09-20-2310-f36283e, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, maintainer=Guillaume Abrioux ) 2024-09-21T04:20:21.389 INFO:teuthology.orchestra.run.smithi089.stdout: 5 247049 9.10 MB/sec execute 540 sec latency 1071.131 ms 2024-09-21T04:20:21.468 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:20:21 smithi089 ceph-mon[64076]: osdmap e145: 8 total, 7 up, 8 in 2024-09-21T04:20:21.468 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:20:21 smithi089 ceph-mon[64076]: from='client.34416 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T04:20:21.468 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:20:21 smithi089 ceph-mon[64076]: from='client.34420 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T04:20:21.581 INFO:teuthology.orchestra.run.smithi089.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-21T04:20:21.581 INFO:teuthology.orchestra.run.smithi089.stdout:alertmanager.smithi089 smithi089 *:9093,9094 running (8m) 108s ago 17m 17.9M - 0.27.0 11f11916f8cd 65dfcbcafee1 2024-09-21T04:20:21.582 INFO:teuthology.orchestra.run.smithi089.stdout:crash.smithi089 smithi089 running (5m) 108s ago 17m 6891k - 19.3.0-5103-gf36283e7 48ed0a2cf89d 6c415a5e173e 2024-09-21T04:20:21.582 INFO:teuthology.orchestra.run.smithi089.stdout:crash.smithi195 smithi195 running (5m) 29s ago 16m 6891k - 19.3.0-5103-gf36283e7 48ed0a2cf89d 78ce05a4e9ab 2024-09-21T04:20:21.582 INFO:teuthology.orchestra.run.smithi089.stdout:grafana.smithi089 smithi089 *:3000 running (8m) 108s ago 17m 47.2M - dad864ee21e9 ec4d6738bce7 2024-09-21T04:20:21.582 INFO:teuthology.orchestra.run.smithi089.stdout:mds.foofs.smithi089.osufws smithi089 running (11m) 108s ago 11m 38.5M - 17.2.0 e1d6a67b021e 664d41859bea 2024-09-21T04:20:21.582 INFO:teuthology.orchestra.run.smithi089.stdout:mds.foofs.smithi195.hijxre smithi195 running (11m) 29s ago 11m 17.6M - 17.2.0 e1d6a67b021e 7bc0c6542d12 2024-09-21T04:20:21.582 INFO:teuthology.orchestra.run.smithi089.stdout:mgr.smithi089.srlyxf smithi089 *:8443,9283,8765 running (7m) 108s ago 18m 519M - 19.3.0-5103-gf36283e7 48ed0a2cf89d 3cbeaaa88241 2024-09-21T04:20:21.582 INFO:teuthology.orchestra.run.smithi089.stdout:mgr.smithi195.cgotzm smithi195 *:8443,9283,8765 running (6m) 29s ago 16m 457M - 19.3.0-5103-gf36283e7 48ed0a2cf89d d3734574cf66 2024-09-21T04:20:21.582 INFO:teuthology.orchestra.run.smithi089.stdout:mon.smithi089 smithi089 running (6m) 108s ago 18m 53.6M 2048M 19.3.0-5103-gf36283e7 48ed0a2cf89d 7dd34c8a6f21 2024-09-21T04:20:21.582 INFO:teuthology.orchestra.run.smithi089.stdout:mon.smithi195 smithi195 running (6m) 29s ago 16m 44.9M 2048M 19.3.0-5103-gf36283e7 48ed0a2cf89d 4255937444e7 2024-09-21T04:20:21.582 INFO:teuthology.orchestra.run.smithi089.stdout:nfs.foo.0.0.smithi089.dcjmqc smithi089 *:2049 running (11m) 108s ago 11m 290M - 4.0 e1d6a67b021e 98550f94e773 2024-09-21T04:20:21.582 INFO:teuthology.orchestra.run.smithi089.stdout:nfs.foo.1.0.smithi195.vhoayp smithi195 *:2049 running (11m) 29s ago 11m 88.6M - 4.0 e1d6a67b021e c60a4040130a 2024-09-21T04:20:21.582 INFO:teuthology.orchestra.run.smithi089.stdout:node-exporter.smithi089 smithi089 *:9100 running (8m) 108s ago 17m 9.84M - 1.7.0 72c9c2088986 7ef0d1588475 2024-09-21T04:20:21.582 INFO:teuthology.orchestra.run.smithi089.stdout:node-exporter.smithi195 smithi195 *:9100 running (7m) 29s ago 16m 11.3M - 1.7.0 72c9c2088986 61eafd61bc6c 2024-09-21T04:20:21.582 INFO:teuthology.orchestra.run.smithi089.stdout:osd.0 smithi089 running (4m) 108s ago 16m 331M 1935M 19.3.0-5103-gf36283e7 48ed0a2cf89d 3df0adea59fb 2024-09-21T04:20:21.583 INFO:teuthology.orchestra.run.smithi089.stdout:osd.1 smithi089 running (3m) 108s ago 15m 265M 1935M 19.3.0-5103-gf36283e7 48ed0a2cf89d da2e49774e7e 2024-09-21T04:20:21.583 INFO:teuthology.orchestra.run.smithi089.stdout:osd.2 smithi089 running (2m) 108s ago 15m 202M 1935M 19.3.0-5103-gf36283e7 48ed0a2cf89d b9f6438dd5b9 2024-09-21T04:20:21.583 INFO:teuthology.orchestra.run.smithi089.stdout:osd.3 smithi089 running (117s) 108s ago 14m 11.5M 1935M 19.3.0-5103-gf36283e7 48ed0a2cf89d d269cd94bd08 2024-09-21T04:20:21.583 INFO:teuthology.orchestra.run.smithi089.stdout:osd.4 smithi195 running (72s) 29s ago 14m 328M 2704M 19.3.0-5103-gf36283e7 48ed0a2cf89d e375f90f95c4 2024-09-21T04:20:21.583 INFO:teuthology.orchestra.run.smithi089.stdout:osd.5 smithi195 running (34s) 29s ago 13m 12.1M 2704M 19.3.0-5103-gf36283e7 48ed0a2cf89d 41de0beb713d 2024-09-21T04:20:21.583 INFO:teuthology.orchestra.run.smithi089.stdout:osd.6 smithi195 running (13m) 29s ago 13m 494M 2704M 17.2.0 e1d6a67b021e 0d7db362cc23 2024-09-21T04:20:21.583 INFO:teuthology.orchestra.run.smithi089.stdout:osd.7 smithi195 running (13m) 29s ago 13m 733M 2704M 17.2.0 e1d6a67b021e 0109b3fea188 2024-09-21T04:20:21.583 INFO:teuthology.orchestra.run.smithi089.stdout:prometheus.smithi089 smithi089 *:9095 running (6m) 108s ago 16m 53.3M - 2.51.0 1d3b7f56885b 8c9f309a6459 2024-09-21T04:20:21.755 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:20:21 smithi195 ceph-mon[49435]: osdmap e145: 8 total, 7 up, 8 in 2024-09-21T04:20:21.755 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:20:21 smithi195 ceph-mon[49435]: from='client.34416 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T04:20:21.755 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:20:21 smithi195 ceph-mon[49435]: from='client.34420 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T04:20:21.755 INFO:journalctl@ceph.osd.6.smithi195.stdout:Sep 21 04:20:21 smithi195 podman[57135]: 2024-09-21 04:20:21.524711457 +0000 UTC m=+0.539645877 container init e2ee93a3976024e5f5fb91e5ce20237fb29c7f411afea2e642745b6662b927c2 (image=quay.ceph.io/ceph-ci/ceph@sha256:2195e5f178be0a9390728e2a14088236d8fe6f82635f5c84418fe37a04457e00, name=ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-6-deactivate, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-guits-main-2024-09-20-2310-f36283e, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=) 2024-09-21T04:20:21.755 INFO:journalctl@ceph.osd.6.smithi195.stdout:Sep 21 04:20:21 smithi195 podman[57135]: 2024-09-21 04:20:21.529426572 +0000 UTC m=+0.544360974 container start e2ee93a3976024e5f5fb91e5ce20237fb29c7f411afea2e642745b6662b927c2 (image=quay.ceph.io/ceph-ci/ceph@sha256:2195e5f178be0a9390728e2a14088236d8fe6f82635f5c84418fe37a04457e00, name=ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-6-deactivate, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, RELEASE=wip-guits-main-2024-09-20-2310-f36283e, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-21T04:20:21.755 INFO:journalctl@ceph.osd.6.smithi195.stdout:Sep 21 04:20:21 smithi195 podman[57135]: 2024-09-21 04:20:21.633252247 +0000 UTC m=+0.648186666 container attach e2ee93a3976024e5f5fb91e5ce20237fb29c7f411afea2e642745b6662b927c2 (image=quay.ceph.io/ceph-ci/ceph@sha256:2195e5f178be0a9390728e2a14088236d8fe6f82635f5c84418fe37a04457e00, name=ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-6-deactivate, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, ceph=True, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, RELEASE=wip-guits-main-2024-09-20-2310-f36283e) 2024-09-21T04:20:22.109 INFO:journalctl@ceph.osd.6.smithi195.stdout:Sep 21 04:20:21 smithi195 podman[57135]: 2024-09-21 04:20:21.752949982 +0000 UTC m=+0.767884401 container died e2ee93a3976024e5f5fb91e5ce20237fb29c7f411afea2e642745b6662b927c2 (image=quay.ceph.io/ceph-ci/ceph@sha256:2195e5f178be0a9390728e2a14088236d8fe6f82635f5c84418fe37a04457e00, name=ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-6-deactivate, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, ceph=True, RELEASE=wip-guits-main-2024-09-20-2310-f36283e, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_CLEAN=True, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2) 2024-09-21T04:20:22.194 INFO:teuthology.orchestra.run.smithi089.stdout:{ 2024-09-21T04:20:22.194 INFO:teuthology.orchestra.run.smithi089.stdout: "mon": { 2024-09-21T04:20:22.194 INFO:teuthology.orchestra.run.smithi089.stdout: "ceph version 19.3.0-5103-gf36283e7 (f36283e7c91cac1c01cb9a7c4ff21bd697dbd4f2) squid (dev)": 2 2024-09-21T04:20:22.194 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2024-09-21T04:20:22.194 INFO:teuthology.orchestra.run.smithi089.stdout: "mgr": { 2024-09-21T04:20:22.194 INFO:teuthology.orchestra.run.smithi089.stdout: "ceph version 19.3.0-5103-gf36283e7 (f36283e7c91cac1c01cb9a7c4ff21bd697dbd4f2) squid (dev)": 2 2024-09-21T04:20:22.194 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2024-09-21T04:20:22.194 INFO:teuthology.orchestra.run.smithi089.stdout: "osd": { 2024-09-21T04:20:22.194 INFO:teuthology.orchestra.run.smithi089.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 1, 2024-09-21T04:20:22.195 INFO:teuthology.orchestra.run.smithi089.stdout: "ceph version 19.3.0-5103-gf36283e7 (f36283e7c91cac1c01cb9a7c4ff21bd697dbd4f2) squid (dev)": 6 2024-09-21T04:20:22.195 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2024-09-21T04:20:22.195 INFO:teuthology.orchestra.run.smithi089.stdout: "mds": { 2024-09-21T04:20:22.195 INFO:teuthology.orchestra.run.smithi089.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-21T04:20:22.195 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2024-09-21T04:20:22.195 INFO:teuthology.orchestra.run.smithi089.stdout: "overall": { 2024-09-21T04:20:22.195 INFO:teuthology.orchestra.run.smithi089.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3, 2024-09-21T04:20:22.195 INFO:teuthology.orchestra.run.smithi089.stdout: "ceph version 19.3.0-5103-gf36283e7 (f36283e7c91cac1c01cb9a7c4ff21bd697dbd4f2) squid (dev)": 10 2024-09-21T04:20:22.195 INFO:teuthology.orchestra.run.smithi089.stdout: } 2024-09-21T04:20:22.195 INFO:teuthology.orchestra.run.smithi089.stdout:} 2024-09-21T04:20:22.389 INFO:teuthology.orchestra.run.smithi089.stdout: 5 247415 9.09 MB/sec execute 541 sec latency 66.201 ms 2024-09-21T04:20:22.485 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:20:22 smithi089 ceph-mon[64076]: from='client.34424 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T04:20:22.485 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:20:22 smithi089 ceph-mon[64076]: pgmap v264: 97 pgs: 10 stale+active+clean, 87 active+clean; 336 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 4.2 KiB/s rd, 2.9 MiB/s wr, 279 op/s 2024-09-21T04:20:22.485 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:20:22 smithi089 ceph-mon[64076]: from='client.? 172.21.15.89:0/2698339453' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T04:20:22.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:20:22 smithi195 ceph-mon[49435]: from='client.34424 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T04:20:22.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:20:22 smithi195 ceph-mon[49435]: pgmap v264: 97 pgs: 10 stale+active+clean, 87 active+clean; 336 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 4.2 KiB/s rd, 2.9 MiB/s wr, 279 op/s 2024-09-21T04:20:22.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:20:22 smithi195 ceph-mon[49435]: from='client.? 172.21.15.89:0/2698339453' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T04:20:22.609 INFO:journalctl@ceph.osd.6.smithi195.stdout:Sep 21 04:20:22 smithi195 podman[57158]: 2024-09-21 04:20:22.474714321 +0000 UTC m=+0.713918600 container remove e2ee93a3976024e5f5fb91e5ce20237fb29c7f411afea2e642745b6662b927c2 (image=quay.ceph.io/ceph-ci/ceph@sha256:2195e5f178be0a9390728e2a14088236d8fe6f82635f5c84418fe37a04457e00, name=ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-6-deactivate, RELEASE=wip-guits-main-2024-09-20-2310-f36283e, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, io.buildah.version=1.37.2, ceph=True, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0) 2024-09-21T04:20:22.609 INFO:journalctl@ceph.osd.6.smithi195.stdout:Sep 21 04:20:22 smithi195 systemd[1]: ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490@osd.6.service: Deactivated successfully. 2024-09-21T04:20:22.610 INFO:journalctl@ceph.osd.6.smithi195.stdout:Sep 21 04:20:22 smithi195 systemd[1]: Stopped Ceph osd.6 for 1b9b79fc-77ce-11ef-baf5-efdc52797490. 2024-09-21T04:20:22.610 INFO:journalctl@ceph.osd.6.smithi195.stdout:Sep 21 04:20:22 smithi195 systemd[1]: ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490@osd.6.service: Consumed 4min 19.733s CPU time. 2024-09-21T04:20:22.754 INFO:teuthology.orchestra.run.smithi089.stdout:{ 2024-09-21T04:20:22.754 INFO:teuthology.orchestra.run.smithi089.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:2195e5f178be0a9390728e2a14088236d8fe6f82635f5c84418fe37a04457e00", 2024-09-21T04:20:22.754 INFO:teuthology.orchestra.run.smithi089.stdout: "in_progress": true, 2024-09-21T04:20:22.754 INFO:teuthology.orchestra.run.smithi089.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-21T04:20:22.754 INFO:teuthology.orchestra.run.smithi089.stdout: "services_complete": [ 2024-09-21T04:20:22.754 INFO:teuthology.orchestra.run.smithi089.stdout: "mgr", 2024-09-21T04:20:22.754 INFO:teuthology.orchestra.run.smithi089.stdout: "crash", 2024-09-21T04:20:22.754 INFO:teuthology.orchestra.run.smithi089.stdout: "mon" 2024-09-21T04:20:22.755 INFO:teuthology.orchestra.run.smithi089.stdout: ], 2024-09-21T04:20:22.755 INFO:teuthology.orchestra.run.smithi089.stdout: "progress": "12/23 daemons upgraded", 2024-09-21T04:20:22.755 INFO:teuthology.orchestra.run.smithi089.stdout: "message": "Currently upgrading osd daemons", 2024-09-21T04:20:22.755 INFO:teuthology.orchestra.run.smithi089.stdout: "is_paused": false 2024-09-21T04:20:22.755 INFO:teuthology.orchestra.run.smithi089.stdout:} 2024-09-21T04:20:23.055 INFO:journalctl@ceph.osd.6.smithi195.stdout:Sep 21 04:20:22 smithi195 systemd[1]: Starting Ceph osd.6 for 1b9b79fc-77ce-11ef-baf5-efdc52797490... 2024-09-21T04:20:23.055 INFO:journalctl@ceph.osd.6.smithi195.stdout:Sep 21 04:20:22 smithi195 podman[57255]: 2024-09-21 04:20:22.811608538 +0000 UTC m=+0.015871518 image pull 48ed0a2cf89d6d0371a2c6185a5869a747265f6abd5ac2a74f00cc32d5628721 quay.ceph.io/ceph-ci/ceph@sha256:2195e5f178be0a9390728e2a14088236d8fe6f82635f5c84418fe37a04457e00 2024-09-21T04:20:23.055 INFO:journalctl@ceph.osd.6.smithi195.stdout:Sep 21 04:20:22 smithi195 podman[57255]: 2024-09-21 04:20:22.934161557 +0000 UTC m=+0.138424533 container create dd6c162d0864f786c3070decee2bde8952cad74d47a06fe27eb2b181024b0de6 (image=quay.ceph.io/ceph-ci/ceph@sha256:2195e5f178be0a9390728e2a14088236d8fe6f82635f5c84418fe37a04457e00, name=ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-6-activate, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, io.buildah.version=1.37.2, RELEASE=wip-guits-main-2024-09-20-2310-f36283e, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.vendor=CentOS) 2024-09-21T04:20:23.313 INFO:journalctl@ceph.osd.6.smithi195.stdout:Sep 21 04:20:23 smithi195 podman[57255]: 2024-09-21 04:20:23.170667092 +0000 UTC m=+0.374930071 container init dd6c162d0864f786c3070decee2bde8952cad74d47a06fe27eb2b181024b0de6 (image=quay.ceph.io/ceph-ci/ceph@sha256:2195e5f178be0a9390728e2a14088236d8fe6f82635f5c84418fe37a04457e00, name=ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-6-activate, org.label-schema.schema-version=1.0, RELEASE=wip-guits-main-2024-09-20-2310-f36283e, ceph=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_CLEAN=True, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS) 2024-09-21T04:20:23.313 INFO:journalctl@ceph.osd.6.smithi195.stdout:Sep 21 04:20:23 smithi195 podman[57255]: 2024-09-21 04:20:23.174199661 +0000 UTC m=+0.378462633 container start dd6c162d0864f786c3070decee2bde8952cad74d47a06fe27eb2b181024b0de6 (image=quay.ceph.io/ceph-ci/ceph@sha256:2195e5f178be0a9390728e2a14088236d8fe6f82635f5c84418fe37a04457e00, name=ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-6-activate, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_CLEAN=True, RELEASE=wip-guits-main-2024-09-20-2310-f36283e, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=) 2024-09-21T04:20:23.313 INFO:journalctl@ceph.osd.6.smithi195.stdout:Sep 21 04:20:23 smithi195 podman[57255]: 2024-09-21 04:20:23.262406982 +0000 UTC m=+0.466669962 container attach dd6c162d0864f786c3070decee2bde8952cad74d47a06fe27eb2b181024b0de6 (image=quay.ceph.io/ceph-ci/ceph@sha256:2195e5f178be0a9390728e2a14088236d8fe6f82635f5c84418fe37a04457e00, name=ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-6-activate, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, RELEASE=wip-guits-main-2024-09-20-2310-f36283e, org.label-schema.license=GPLv2) 2024-09-21T04:20:23.389 INFO:teuthology.orchestra.run.smithi089.stdout: 5 247742 9.09 MB/sec execute 542 sec latency 37.703 ms 2024-09-21T04:20:23.441 INFO:teuthology.orchestra.run.smithi089.stdout:HEALTH_WARN 1 osds down 2024-09-21T04:20:23.441 INFO:teuthology.orchestra.run.smithi089.stdout:[WRN] OSD_DOWN: 1 osds down 2024-09-21T04:20:23.442 INFO:teuthology.orchestra.run.smithi089.stdout: osd.6 (root=default,host=smithi195) is down 2024-09-21T04:20:23.512 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:20:23 smithi089 ceph-mon[64076]: from='client.34432 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T04:20:23.512 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:20:23 smithi089 ceph-mon[64076]: from='client.? 172.21.15.89:0/465312799' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-21T04:20:23.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:20:23 smithi195 ceph-mon[49435]: from='client.34432 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T04:20:23.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:20:23 smithi195 ceph-mon[49435]: from='client.? 172.21.15.89:0/465312799' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-21T04:20:23.609 INFO:journalctl@ceph.osd.6.smithi195.stdout:Sep 21 04:20:23 smithi195 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-6-activate[57270]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T04:20:23.610 INFO:journalctl@ceph.osd.6.smithi195.stdout:Sep 21 04:20:23 smithi195 bash[57255]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T04:20:23.610 INFO:journalctl@ceph.osd.6.smithi195.stdout:Sep 21 04:20:23 smithi195 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-6-activate[57270]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T04:20:23.610 INFO:journalctl@ceph.osd.6.smithi195.stdout:Sep 21 04:20:23 smithi195 bash[57255]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T04:20:24.389 INFO:teuthology.orchestra.run.smithi089.stdout: 5 247931 9.08 MB/sec execute 543 sec latency 40.222 ms 2024-09-21T04:20:24.547 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:20:24 smithi089 ceph-mon[64076]: pgmap v265: 97 pgs: 5 active+undersized, 5 stale+active+clean, 17 active+undersized+degraded, 70 active+clean; 336 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 4.1 MiB/s wr, 411 op/s; 132/1623 objects degraded (8.133%) 2024-09-21T04:20:24.859 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:20:24 smithi195 ceph-mon[49435]: pgmap v265: 97 pgs: 5 active+undersized, 5 stale+active+clean, 17 active+undersized+degraded, 70 active+clean; 336 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 4.1 MiB/s wr, 411 op/s; 132/1623 objects degraded (8.133%) 2024-09-21T04:20:25.389 INFO:teuthology.orchestra.run.smithi089.stdout: 5 248240 9.09 MB/sec execute 544 sec latency 47.001 ms 2024-09-21T04:20:25.569 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:20:25 smithi089 ceph-mon[64076]: Health check failed: Degraded data redundancy: 132/1623 objects degraded (8.133%), 17 pgs degraded (PG_DEGRADED) 2024-09-21T04:20:25.859 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:20:25 smithi195 ceph-mon[49435]: Health check failed: Degraded data redundancy: 132/1623 objects degraded (8.133%), 17 pgs degraded (PG_DEGRADED) 2024-09-21T04:20:26.389 INFO:teuthology.orchestra.run.smithi089.stdout: 5 248544 9.09 MB/sec execute 545 sec latency 58.317 ms 2024-09-21T04:20:26.524 INFO:journalctl@ceph.osd.6.smithi195.stdout:Sep 21 04:20:26 smithi195 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-6-activate[57270]: --> Failed to activate via raw: did not find any matching OSD to activate 2024-09-21T04:20:26.525 INFO:journalctl@ceph.osd.6.smithi195.stdout:Sep 21 04:20:26 smithi195 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-6-activate[57270]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T04:20:26.525 INFO:journalctl@ceph.osd.6.smithi195.stdout:Sep 21 04:20:26 smithi195 bash[57255]: --> Failed to activate via raw: did not find any matching OSD to activate 2024-09-21T04:20:26.525 INFO:journalctl@ceph.osd.6.smithi195.stdout:Sep 21 04:20:26 smithi195 bash[57255]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T04:20:26.525 INFO:journalctl@ceph.osd.6.smithi195.stdout:Sep 21 04:20:26 smithi195 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-6-activate[57270]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T04:20:26.525 INFO:journalctl@ceph.osd.6.smithi195.stdout:Sep 21 04:20:26 smithi195 bash[57255]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T04:20:26.525 INFO:journalctl@ceph.osd.6.smithi195.stdout:Sep 21 04:20:26 smithi195 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-6-activate[57270]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-6 2024-09-21T04:20:26.525 INFO:journalctl@ceph.osd.6.smithi195.stdout:Sep 21 04:20:26 smithi195 bash[57255]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-6 2024-09-21T04:20:26.525 INFO:journalctl@ceph.osd.6.smithi195.stdout:Sep 21 04:20:26 smithi195 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-6-activate[57270]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_2 --path /var/lib/ceph/osd/ceph-6 --no-mon-config 2024-09-21T04:20:26.525 INFO:journalctl@ceph.osd.6.smithi195.stdout:Sep 21 04:20:26 smithi195 bash[57255]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_2 --path /var/lib/ceph/osd/ceph-6 --no-mon-config 2024-09-21T04:20:26.604 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:20:26 smithi089 ceph-mon[64076]: pgmap v266: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 334 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 1.5 KiB/s rd, 3.6 MiB/s wr, 423 op/s; 186/1638 objects degraded (11.355%) 2024-09-21T04:20:26.859 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:20:26 smithi195 ceph-mon[49435]: pgmap v266: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 334 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 1.5 KiB/s rd, 3.6 MiB/s wr, 423 op/s; 186/1638 objects degraded (11.355%) 2024-09-21T04:20:26.859 INFO:journalctl@ceph.osd.6.smithi195.stdout:Sep 21 04:20:26 smithi195 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-6-activate[57270]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_2 /var/lib/ceph/osd/ceph-6/block 2024-09-21T04:20:26.859 INFO:journalctl@ceph.osd.6.smithi195.stdout:Sep 21 04:20:26 smithi195 bash[57255]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_2 /var/lib/ceph/osd/ceph-6/block 2024-09-21T04:20:26.859 INFO:journalctl@ceph.osd.6.smithi195.stdout:Sep 21 04:20:26 smithi195 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-6-activate[57270]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-6/block 2024-09-21T04:20:26.860 INFO:journalctl@ceph.osd.6.smithi195.stdout:Sep 21 04:20:26 smithi195 bash[57255]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-6/block 2024-09-21T04:20:26.860 INFO:journalctl@ceph.osd.6.smithi195.stdout:Sep 21 04:20:26 smithi195 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-6-activate[57270]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-09-21T04:20:26.860 INFO:journalctl@ceph.osd.6.smithi195.stdout:Sep 21 04:20:26 smithi195 bash[57255]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-09-21T04:20:26.860 INFO:journalctl@ceph.osd.6.smithi195.stdout:Sep 21 04:20:26 smithi195 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-6-activate[57270]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-6 2024-09-21T04:20:26.860 INFO:journalctl@ceph.osd.6.smithi195.stdout:Sep 21 04:20:26 smithi195 bash[57255]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-6 2024-09-21T04:20:26.860 INFO:journalctl@ceph.osd.6.smithi195.stdout:Sep 21 04:20:26 smithi195 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-6-activate[57270]: --> ceph-volume lvm activate successful for osd ID: 6 2024-09-21T04:20:26.860 INFO:journalctl@ceph.osd.6.smithi195.stdout:Sep 21 04:20:26 smithi195 bash[57255]: --> ceph-volume lvm activate successful for osd ID: 6 2024-09-21T04:20:26.860 INFO:journalctl@ceph.osd.6.smithi195.stdout:Sep 21 04:20:26 smithi195 podman[57255]: 2024-09-21 04:20:26.63202776 +0000 UTC m=+3.836290738 container died dd6c162d0864f786c3070decee2bde8952cad74d47a06fe27eb2b181024b0de6 (image=quay.ceph.io/ceph-ci/ceph@sha256:2195e5f178be0a9390728e2a14088236d8fe6f82635f5c84418fe37a04457e00, name=ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-6-activate, org.label-schema.build-date=20240731, ceph=True, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, RELEASE=wip-guits-main-2024-09-20-2310-f36283e, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=) 2024-09-21T04:20:27.389 INFO:teuthology.orchestra.run.smithi089.stdout: 5 248902 9.09 MB/sec execute 546 sec latency 53.534 ms 2024-09-21T04:20:27.749 INFO:journalctl@ceph.osd.6.smithi195.stdout:Sep 21 04:20:27 smithi195 podman[57480]: 2024-09-21 04:20:27.394165333 +0000 UTC m=+0.756307392 container remove dd6c162d0864f786c3070decee2bde8952cad74d47a06fe27eb2b181024b0de6 (image=quay.ceph.io/ceph-ci/ceph@sha256:2195e5f178be0a9390728e2a14088236d8fe6f82635f5c84418fe37a04457e00, name=ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-6-activate, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, RELEASE=wip-guits-main-2024-09-20-2310-f36283e, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, ceph=True) 2024-09-21T04:20:27.749 INFO:journalctl@ceph.osd.6.smithi195.stdout:Sep 21 04:20:27 smithi195 podman[57540]: 2024-09-21 04:20:27.50595635 +0000 UTC m=+0.016287717 image pull 48ed0a2cf89d6d0371a2c6185a5869a747265f6abd5ac2a74f00cc32d5628721 quay.ceph.io/ceph-ci/ceph@sha256:2195e5f178be0a9390728e2a14088236d8fe6f82635f5c84418fe37a04457e00 2024-09-21T04:20:27.749 INFO:journalctl@ceph.osd.6.smithi195.stdout:Sep 21 04:20:27 smithi195 podman[57540]: 2024-09-21 04:20:27.628561565 +0000 UTC m=+0.138892917 container create a41d0c0574ab7012ec0367c29c113be073f9c7c10e8132e59d41c848df04842a (image=quay.ceph.io/ceph-ci/ceph@sha256:2195e5f178be0a9390728e2a14088236d8fe6f82635f5c84418fe37a04457e00, name=ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-6, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-guits-main-2024-09-20-2310-f36283e, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, GIT_CLEAN=True) 2024-09-21T04:20:28.109 INFO:journalctl@ceph.osd.6.smithi195.stdout:Sep 21 04:20:27 smithi195 podman[57540]: 2024-09-21 04:20:27.873385232 +0000 UTC m=+0.383716585 container init a41d0c0574ab7012ec0367c29c113be073f9c7c10e8132e59d41c848df04842a (image=quay.ceph.io/ceph-ci/ceph@sha256:2195e5f178be0a9390728e2a14088236d8fe6f82635f5c84418fe37a04457e00, name=ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-6, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_CLEAN=True, RELEASE=wip-guits-main-2024-09-20-2310-f36283e, io.buildah.version=1.37.2, ceph=True) 2024-09-21T04:20:28.109 INFO:journalctl@ceph.osd.6.smithi195.stdout:Sep 21 04:20:27 smithi195 podman[57540]: 2024-09-21 04:20:27.877049494 +0000 UTC m=+0.387380847 container start a41d0c0574ab7012ec0367c29c113be073f9c7c10e8132e59d41c848df04842a (image=quay.ceph.io/ceph-ci/ceph@sha256:2195e5f178be0a9390728e2a14088236d8fe6f82635f5c84418fe37a04457e00, name=ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-6, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, RELEASE=wip-guits-main-2024-09-20-2310-f36283e, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_BRANCH=HEAD) 2024-09-21T04:20:28.109 INFO:journalctl@ceph.osd.6.smithi195.stdout:Sep 21 04:20:27 smithi195 ceph-osd[57558]: -- 172.21.15.195:0/718444183 <== mon.1 v2:172.21.15.195:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x5599339da1e0 con 0x559933966000 2024-09-21T04:20:28.109 INFO:journalctl@ceph.osd.6.smithi195.stdout:Sep 21 04:20:27 smithi195 bash[57540]: a41d0c0574ab7012ec0367c29c113be073f9c7c10e8132e59d41c848df04842a 2024-09-21T04:20:28.110 INFO:journalctl@ceph.osd.6.smithi195.stdout:Sep 21 04:20:27 smithi195 systemd[1]: Started Ceph osd.6 for 1b9b79fc-77ce-11ef-baf5-efdc52797490. 2024-09-21T04:20:28.389 INFO:teuthology.orchestra.run.smithi089.stdout: 5 249115 9.09 MB/sec execute 547 sec latency 57.072 ms 2024-09-21T04:20:29.209 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:20:29 smithi089 ceph-mon[64076]: pgmap v267: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 326 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 1.6 KiB/s rd, 3.3 MiB/s wr, 386 op/s; 187/1650 objects degraded (11.333%) 2024-09-21T04:20:29.209 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:20:29 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:20:29.209 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:20:29 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:20:29.209 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:20:29 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T04:20:29.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:20:29 smithi195 ceph-mon[49435]: pgmap v267: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 326 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 1.6 KiB/s rd, 3.3 MiB/s wr, 386 op/s; 187/1650 objects degraded (11.333%) 2024-09-21T04:20:29.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:20:29 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:20:29.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:20:29 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:20:29.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:20:29 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T04:20:29.389 INFO:teuthology.orchestra.run.smithi089.stdout: 5 249288 9.08 MB/sec execute 548 sec latency 56.677 ms 2024-09-21T04:20:29.859 INFO:journalctl@ceph.osd.6.smithi195.stdout:Sep 21 04:20:29 smithi195 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-6[57554]: 2024-09-21T04:20:29.524+0000 7f987b50e640 -1 Falling back to public interface 2024-09-21T04:20:30.234 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:20:30 smithi089 ceph-mon[64076]: Health check update: Degraded data redundancy: 187/1650 objects degraded (11.333%), 25 pgs degraded (PG_DEGRADED) 2024-09-21T04:20:30.351 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:20:30 smithi195 ceph-mon[49435]: Health check update: Degraded data redundancy: 187/1650 objects degraded (11.333%), 25 pgs degraded (PG_DEGRADED) 2024-09-21T04:20:30.389 INFO:teuthology.orchestra.run.smithi089.stdout: 5 249548 9.08 MB/sec execute 549 sec latency 62.728 ms 2024-09-21T04:20:31.238 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:20:31 smithi089 ceph-mon[64076]: pgmap v268: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 330 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 4.3 KiB/s rd, 3.8 MiB/s wr, 373 op/s; 171/1614 objects degraded (10.595%) 2024-09-21T04:20:31.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:20:31 smithi195 ceph-mon[49435]: pgmap v268: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 330 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 4.3 KiB/s rd, 3.8 MiB/s wr, 373 op/s; 171/1614 objects degraded (10.595%) 2024-09-21T04:20:31.389 INFO:teuthology.orchestra.run.smithi089.stdout: 5 249962 9.08 MB/sec execute 550 sec latency 67.346 ms 2024-09-21T04:20:32.389 INFO:teuthology.orchestra.run.smithi089.stdout: 5 250221 9.07 MB/sec execute 551 sec latency 62.071 ms 2024-09-21T04:20:32.524 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:20:32 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:20:32.525 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:20:32 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:20:32.525 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:20:32 smithi089 ceph-mon[64076]: pgmap v269: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 335 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 3.7 KiB/s rd, 3.9 MiB/s wr, 433 op/s; 171/1617 objects degraded (10.575%) 2024-09-21T04:20:32.525 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:20:32 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:20:32.525 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:20:32 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:20:32.525 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:20:32 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:20:32.525 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:20:32 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T04:20:32.812 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:20:32 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:20:32.812 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:20:32 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:20:32.812 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:20:32 smithi195 ceph-mon[49435]: pgmap v269: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 335 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 3.7 KiB/s rd, 3.9 MiB/s wr, 433 op/s; 171/1617 objects degraded (10.575%) 2024-09-21T04:20:32.812 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:20:32 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:20:32.812 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:20:32 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:20:32.812 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:20:32 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:20:32.812 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:20:32 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T04:20:33.389 INFO:teuthology.orchestra.run.smithi089.stdout: 5 250559 9.08 MB/sec execute 552 sec latency 60.284 ms 2024-09-21T04:20:34.389 INFO:teuthology.orchestra.run.smithi089.stdout: 5 250913 9.07 MB/sec execute 553 sec latency 74.827 ms 2024-09-21T04:20:35.081 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:20:35 smithi089 ceph-mon[64076]: pgmap v270: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 329 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 6.8 KiB/s rd, 4.5 MiB/s wr, 421 op/s; 176/1632 objects degraded (10.784%) 2024-09-21T04:20:35.081 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:20:35 smithi089 ceph-mon[64076]: Health check update: Degraded data redundancy: 176/1632 objects degraded (10.784%), 25 pgs degraded (PG_DEGRADED) 2024-09-21T04:20:35.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:20:35 smithi195 ceph-mon[49435]: pgmap v270: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 329 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 6.8 KiB/s rd, 4.5 MiB/s wr, 421 op/s; 176/1632 objects degraded (10.784%) 2024-09-21T04:20:35.360 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:20:35 smithi195 ceph-mon[49435]: Health check update: Degraded data redundancy: 176/1632 objects degraded (10.784%), 25 pgs degraded (PG_DEGRADED) 2024-09-21T04:20:35.389 INFO:teuthology.orchestra.run.smithi089.stdout: 5 251320 9.07 MB/sec execute 554 sec latency 35.976 ms 2024-09-21T04:20:36.109 INFO:journalctl@ceph.osd.6.smithi195.stdout:Sep 21 04:20:35 smithi195 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-6[57554]: 2024-09-21T04:20:35.792+0000 7f987b50e640 -1 osd.6 0 read_superblock omap replica is missing. 2024-09-21T04:20:36.389 INFO:teuthology.orchestra.run.smithi089.stdout: 5 251522 9.06 MB/sec execute 555 sec latency 56.370 ms 2024-09-21T04:20:37.171 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:20:37 smithi089 ceph-mon[64076]: pgmap v271: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 331 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 4.0 MiB/s wr, 406 op/s; 175/1614 objects degraded (10.843%) 2024-09-21T04:20:37.171 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:20:37 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:20:37.171 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:20:37 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:20:37.171 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:20:37 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T04:20:37.171 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:20:37 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T04:20:37.171 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:20:37 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:20:37.171 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:20:37 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:20:37.172 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:20:37 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T04:20:37.172 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:20:37 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "versions"} : dispatch 2024-09-21T04:20:37.172 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:20:37 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "versions"} : dispatch 2024-09-21T04:20:37.172 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:20:37 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "versions"} : dispatch 2024-09-21T04:20:37.172 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:20:37 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16} : dispatch 2024-09-21T04:20:37.172 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:20:37 smithi089 ceph-mon[64076]: from='osd.6 [v2:172.21.15.195:6816/3477328135,v1:172.21.15.195:6817/3477328135]' entity='osd.6' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]} : dispatch 2024-09-21T04:20:37.172 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:20:37 smithi089 ceph-mon[64076]: from='osd.6 ' entity='osd.6' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]} : dispatch 2024-09-21T04:20:37.359 INFO:journalctl@ceph.osd.6.smithi195.stdout:Sep 21 04:20:36 smithi195 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-6[57554]: 2024-09-21T04:20:36.954+0000 7f987b50e640 -1 osd.6 143 log_to_monitors true 2024-09-21T04:20:37.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:20:37 smithi195 ceph-mon[49435]: pgmap v271: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 331 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 4.0 MiB/s wr, 406 op/s; 175/1614 objects degraded (10.843%) 2024-09-21T04:20:37.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:20:37 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:20:37.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:20:37 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:20:37.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:20:37 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T04:20:37.360 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:20:37 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T04:20:37.360 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:20:37 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:20:37.360 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:20:37 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:20:37.360 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:20:37 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T04:20:37.360 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:20:37 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "versions"} : dispatch 2024-09-21T04:20:37.360 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:20:37 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "versions"} : dispatch 2024-09-21T04:20:37.360 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:20:37 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "versions"} : dispatch 2024-09-21T04:20:37.360 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:20:37 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16} : dispatch 2024-09-21T04:20:37.360 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:20:37 smithi195 ceph-mon[49435]: from='osd.6 [v2:172.21.15.195:6816/3477328135,v1:172.21.15.195:6817/3477328135]' entity='osd.6' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]} : dispatch 2024-09-21T04:20:37.360 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:20:37 smithi195 ceph-mon[49435]: from='osd.6 ' entity='osd.6' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]} : dispatch 2024-09-21T04:20:37.389 INFO:teuthology.orchestra.run.smithi089.stdout: 5 251878 9.08 MB/sec execute 556 sec latency 39.889 ms 2024-09-21T04:20:38.085 INFO:journalctl@ceph.osd.6.smithi195.stdout:Sep 21 04:20:37 smithi195 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-6[57554]: 2024-09-21T04:20:37.619+0000 7f986f2b0640 -1 osd.6 143 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-21T04:20:38.219 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:20:38 smithi089 ceph-mon[64076]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-09-21T04:20:38.219 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:20:38 smithi089 ceph-mon[64076]: Upgrade: unsafe to stop osd(s) at this time (9 PGs are or would become offline) 2024-09-21T04:20:38.219 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:20:38 smithi089 ceph-mon[64076]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-09-21T04:20:38.219 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:20:38 smithi089 ceph-mon[64076]: osdmap e146: 8 total, 7 up, 8 in 2024-09-21T04:20:38.219 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:20:38 smithi089 ceph-mon[64076]: from='osd.6 [v2:172.21.15.195:6816/3477328135,v1:172.21.15.195:6817/3477328135]' entity='osd.6' cmd={"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi195", "root=default"]} : dispatch 2024-09-21T04:20:38.219 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:20:38 smithi089 ceph-mon[64076]: from='osd.6 ' entity='osd.6' cmd={"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi195", "root=default"]} : dispatch 2024-09-21T04:20:38.219 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:20:38 smithi089 ceph-mon[64076]: from='osd.6 ' entity='osd.6' 2024-09-21T04:20:38.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:20:38 smithi195 ceph-mon[49435]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-09-21T04:20:38.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:20:38 smithi195 ceph-mon[49435]: Upgrade: unsafe to stop osd(s) at this time (9 PGs are or would become offline) 2024-09-21T04:20:38.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:20:38 smithi195 ceph-mon[49435]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-09-21T04:20:38.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:20:38 smithi195 ceph-mon[49435]: osdmap e146: 8 total, 7 up, 8 in 2024-09-21T04:20:38.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:20:38 smithi195 ceph-mon[49435]: from='osd.6 [v2:172.21.15.195:6816/3477328135,v1:172.21.15.195:6817/3477328135]' entity='osd.6' cmd={"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi195", "root=default"]} : dispatch 2024-09-21T04:20:38.360 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:20:38 smithi195 ceph-mon[49435]: from='osd.6 ' entity='osd.6' cmd={"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi195", "root=default"]} : dispatch 2024-09-21T04:20:38.360 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:20:38 smithi195 ceph-mon[49435]: from='osd.6 ' entity='osd.6' 2024-09-21T04:20:38.398 INFO:teuthology.orchestra.run.smithi089.stdout: 5 252217 9.07 MB/sec execute 557 sec latency 84.487 ms 2024-09-21T04:20:39.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:20:39 smithi195 ceph-mon[49435]: pgmap v273: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 330 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 7.0 KiB/s rd, 4.2 MiB/s wr, 398 op/s; 175/1584 objects degraded (11.048%) 2024-09-21T04:20:39.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:20:39 smithi195 ceph-mon[49435]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-21T04:20:39.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:20:39 smithi195 ceph-mon[49435]: osd.6 [v2:172.21.15.195:6816/3477328135,v1:172.21.15.195:6817/3477328135] boot 2024-09-21T04:20:39.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:20:39 smithi195 ceph-mon[49435]: osdmap e147: 8 total, 8 up, 8 in 2024-09-21T04:20:39.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:20:39 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "osd metadata", "id": 6} : dispatch 2024-09-21T04:20:39.398 INFO:teuthology.orchestra.run.smithi089.stdout: 5 252290 9.05 MB/sec execute 558 sec latency 1062.269 ms 2024-09-21T04:20:39.407 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:20:39 smithi089 ceph-mon[64076]: pgmap v273: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 330 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 7.0 KiB/s rd, 4.2 MiB/s wr, 398 op/s; 175/1584 objects degraded (11.048%) 2024-09-21T04:20:39.407 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:20:39 smithi089 ceph-mon[64076]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-21T04:20:39.407 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:20:39 smithi089 ceph-mon[64076]: osd.6 [v2:172.21.15.195:6816/3477328135,v1:172.21.15.195:6817/3477328135] boot 2024-09-21T04:20:39.407 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:20:39 smithi089 ceph-mon[64076]: osdmap e147: 8 total, 8 up, 8 in 2024-09-21T04:20:39.407 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:20:39 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "osd metadata", "id": 6} : dispatch 2024-09-21T04:20:40.398 INFO:teuthology.orchestra.run.smithi089.stdout: 5 252356 9.04 MB/sec execute 559 sec latency 2062.300 ms 2024-09-21T04:20:40.594 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:20:40 smithi089 ceph-mon[64076]: osdmap e148: 8 total, 8 up, 8 in 2024-09-21T04:20:40.594 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:20:40 smithi089 ceph-mon[64076]: Health check update: Degraded data redundancy: 175/1584 objects degraded (11.048%), 25 pgs degraded (PG_DEGRADED) 2024-09-21T04:20:40.594 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:20:40 smithi089 ceph-mon[64076]: pgmap v276: 97 pgs: 11 peering, 4 active+undersized, 20 active+undersized+degraded, 62 active+clean; 335 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 5.0 KiB/s rd, 4.4 MiB/s wr, 536 op/s; 131/1647 objects degraded (7.954%) 2024-09-21T04:20:40.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:20:40 smithi195 ceph-mon[49435]: osdmap e148: 8 total, 8 up, 8 in 2024-09-21T04:20:40.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:20:40 smithi195 ceph-mon[49435]: Health check update: Degraded data redundancy: 175/1584 objects degraded (11.048%), 25 pgs degraded (PG_DEGRADED) 2024-09-21T04:20:40.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:20:40 smithi195 ceph-mon[49435]: pgmap v276: 97 pgs: 11 peering, 4 active+undersized, 20 active+undersized+degraded, 62 active+clean; 335 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 5.0 KiB/s rd, 4.4 MiB/s wr, 536 op/s; 131/1647 objects degraded (7.954%) 2024-09-21T04:20:41.349 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:20:41 smithi089 ceph-mon[64076]: osdmap e149: 8 total, 8 up, 8 in 2024-09-21T04:20:41.398 INFO:teuthology.orchestra.run.smithi089.stdout: 5 252592 9.04 MB/sec execute 560 sec latency 2237.930 ms 2024-09-21T04:20:41.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:20:41 smithi195 ceph-mon[49435]: osdmap e149: 8 total, 8 up, 8 in 2024-09-21T04:20:42.375 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:20:42 smithi089 ceph-mon[64076]: osdmap e150: 8 total, 8 up, 8 in 2024-09-21T04:20:42.375 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:20:42 smithi089 ceph-mon[64076]: pgmap v279: 97 pgs: 11 peering, 4 active+undersized, 20 active+undersized+degraded, 62 active+clean; 332 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 7.0 KiB/s rd, 4.1 MiB/s wr, 480 op/s; 131/1638 objects degraded (7.998%) 2024-09-21T04:20:42.398 INFO:teuthology.orchestra.run.smithi089.stdout: 5 252864 9.05 MB/sec execute 561 sec latency 713.393 ms 2024-09-21T04:20:42.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:20:42 smithi195 ceph-mon[49435]: osdmap e150: 8 total, 8 up, 8 in 2024-09-21T04:20:42.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:20:42 smithi195 ceph-mon[49435]: pgmap v279: 97 pgs: 11 peering, 4 active+undersized, 20 active+undersized+degraded, 62 active+clean; 332 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 7.0 KiB/s rd, 4.1 MiB/s wr, 480 op/s; 131/1638 objects degraded (7.998%) 2024-09-21T04:20:43.397 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:20:43 smithi089 ceph-mon[64076]: osdmap e151: 8 total, 8 up, 8 in 2024-09-21T04:20:43.398 INFO:teuthology.orchestra.run.smithi089.stdout: 5 253113 9.04 MB/sec execute 562 sec latency 942.595 ms 2024-09-21T04:20:43.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:20:43 smithi195 ceph-mon[49435]: osdmap e151: 8 total, 8 up, 8 in 2024-09-21T04:20:44.398 INFO:teuthology.orchestra.run.smithi089.stdout: 5 253489 9.04 MB/sec execute 563 sec latency 33.790 ms 2024-09-21T04:20:44.406 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:20:44 smithi089 ceph-mon[64076]: osdmap e152: 8 total, 8 up, 8 in 2024-09-21T04:20:44.406 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:20:44 smithi089 ceph-mon[64076]: pgmap v282: 97 pgs: 11 peering, 86 active+clean; 332 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 3.2 KiB/s rd, 3.0 MiB/s wr, 186 op/s; 2.6 MiB/s, 67 keys/s, 42 objects/s recovering 2024-09-21T04:20:44.859 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:20:44 smithi195 ceph-mon[49435]: osdmap e152: 8 total, 8 up, 8 in 2024-09-21T04:20:44.859 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:20:44 smithi195 ceph-mon[49435]: pgmap v282: 97 pgs: 11 peering, 86 active+clean; 332 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 3.2 KiB/s rd, 3.0 MiB/s wr, 186 op/s; 2.6 MiB/s, 67 keys/s, 42 objects/s recovering 2024-09-21T04:20:45.398 INFO:teuthology.orchestra.run.smithi089.stdout: 5 253831 9.04 MB/sec execute 564 sec latency 38.677 ms 2024-09-21T04:20:45.420 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:20:45 smithi089 ceph-mon[64076]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 131/1638 objects degraded (7.998%), 20 pgs degraded) 2024-09-21T04:20:45.420 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:20:45 smithi089 ceph-mon[64076]: Cluster is now healthy 2024-09-21T04:20:45.859 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:20:45 smithi195 ceph-mon[49435]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 131/1638 objects degraded (7.998%), 20 pgs degraded) 2024-09-21T04:20:45.859 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:20:45 smithi195 ceph-mon[49435]: Cluster is now healthy 2024-09-21T04:20:46.398 INFO:teuthology.orchestra.run.smithi089.stdout: 5 254142 9.04 MB/sec execute 565 sec latency 47.929 ms 2024-09-21T04:20:46.438 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:20:46 smithi089 ceph-mon[64076]: pgmap v283: 97 pgs: 3 peering, 94 active+clean; 331 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 2.3 KiB/s rd, 3.1 MiB/s wr, 285 op/s; 1.9 MiB/s, 47 keys/s, 36 objects/s recovering 2024-09-21T04:20:46.859 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:20:46 smithi195 ceph-mon[49435]: pgmap v283: 97 pgs: 3 peering, 94 active+clean; 331 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 2.3 KiB/s rd, 3.1 MiB/s wr, 285 op/s; 1.9 MiB/s, 47 keys/s, 36 objects/s recovering 2024-09-21T04:20:47.398 INFO:teuthology.orchestra.run.smithi089.stdout: 5 254557 9.03 MB/sec execute 566 sec latency 73.318 ms 2024-09-21T04:20:48.398 INFO:teuthology.orchestra.run.smithi089.stdout: 5 254931 9.03 MB/sec execute 567 sec latency 38.108 ms 2024-09-21T04:20:48.467 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:20:48 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:20:48.467 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:20:48 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T04:20:48.467 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:20:48 smithi089 ceph-mon[64076]: pgmap v284: 97 pgs: 97 active+clean; 334 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 7.9 KiB/s rd, 4.2 MiB/s wr, 402 op/s; 1.6 MiB/s, 40 keys/s, 31 objects/s recovering 2024-09-21T04:20:48.859 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:20:48 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:20:48.859 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:20:48 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T04:20:48.859 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:20:48 smithi195 ceph-mon[49435]: pgmap v284: 97 pgs: 97 active+clean; 334 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 7.9 KiB/s rd, 4.2 MiB/s wr, 402 op/s; 1.6 MiB/s, 40 keys/s, 31 objects/s recovering 2024-09-21T04:20:49.398 INFO:teuthology.orchestra.run.smithi089.stdout: 5 255272 9.03 MB/sec execute 568 sec latency 64.785 ms 2024-09-21T04:20:50.398 INFO:teuthology.orchestra.run.smithi089.stdout: 5 255607 9.03 MB/sec execute 569 sec latency 62.408 ms 2024-09-21T04:20:51.251 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:20:51 smithi089 ceph-mon[64076]: pgmap v285: 97 pgs: 97 active+clean; 335 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 6.6 KiB/s rd, 5.1 MiB/s wr, 518 op/s; 1.4 MiB/s, 33 keys/s, 25 objects/s recovering 2024-09-21T04:20:51.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:20:51 smithi195 ceph-mon[49435]: pgmap v285: 97 pgs: 97 active+clean; 335 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 6.6 KiB/s rd, 5.1 MiB/s wr, 518 op/s; 1.4 MiB/s, 33 keys/s, 25 objects/s recovering 2024-09-21T04:20:51.398 INFO:teuthology.orchestra.run.smithi089.stdout: 5 255996 9.05 MB/sec execute 570 sec latency 51.689 ms 2024-09-21T04:20:52.264 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:20:52 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16} : dispatch 2024-09-21T04:20:52.333 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:20:52 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16} : dispatch 2024-09-21T04:20:52.398 INFO:teuthology.orchestra.run.smithi089.stdout: 5 256350 9.05 MB/sec execute 571 sec latency 71.336 ms 2024-09-21T04:20:53.294 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:20:53 smithi089 ceph-mon[64076]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-09-21T04:20:53.294 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:20:53 smithi089 ceph-mon[64076]: Upgrade: osd.7 is safe to restart 2024-09-21T04:20:53.294 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:20:53 smithi089 ceph-mon[64076]: pgmap v286: 97 pgs: 97 active+clean; 330 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 5.5 KiB/s rd, 4.1 MiB/s wr, 422 op/s; 1022 KiB/s, 27 keys/s, 17 objects/s recovering 2024-09-21T04:20:53.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:20:53 smithi195 ceph-mon[49435]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-09-21T04:20:53.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:20:53 smithi195 ceph-mon[49435]: Upgrade: osd.7 is safe to restart 2024-09-21T04:20:53.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:20:53 smithi195 ceph-mon[49435]: pgmap v286: 97 pgs: 97 active+clean; 330 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 5.5 KiB/s rd, 4.1 MiB/s wr, 422 op/s; 1022 KiB/s, 27 keys/s, 17 objects/s recovering 2024-09-21T04:20:53.398 INFO:teuthology.orchestra.run.smithi089.stdout: 5 256644 9.05 MB/sec execute 572 sec latency 60.964 ms 2024-09-21T04:20:53.980 INFO:teuthology.orchestra.run.smithi089.stdout:true 2024-09-21T04:20:54.332 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:20:54 smithi089 ceph-mon[64076]: Upgrade: Updating osd.7 2024-09-21T04:20:54.332 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:20:54 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:20:54.333 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:20:54 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "auth get", "entity": "osd.7"} : dispatch 2024-09-21T04:20:54.333 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:20:54 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T04:20:54.333 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:20:54 smithi089 ceph-mon[64076]: Deploying daemon osd.7 on smithi195 2024-09-21T04:20:54.333 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:20:54 smithi089 ceph-mon[64076]: from='client.34444 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T04:20:54.333 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:20:54 smithi089 ceph-mon[64076]: pgmap v287: 97 pgs: 97 active+clean; 332 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 6.9 KiB/s rd, 5.3 MiB/s wr, 461 op/s; 169 KiB/s, 25 keys/s, 5 objects/s recovering 2024-09-21T04:20:54.399 INFO:teuthology.orchestra.run.smithi089.stdout: 5 256966 9.04 MB/sec execute 573 sec latency 49.037 ms 2024-09-21T04:20:54.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:20:54 smithi195 ceph-mon[49435]: Upgrade: Updating osd.7 2024-09-21T04:20:54.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:20:54 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:20:54.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:20:54 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "auth get", "entity": "osd.7"} : dispatch 2024-09-21T04:20:54.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:20:54 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T04:20:54.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:20:54 smithi195 ceph-mon[49435]: Deploying daemon osd.7 on smithi195 2024-09-21T04:20:54.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:20:54 smithi195 ceph-mon[49435]: from='client.34444 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T04:20:54.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:20:54 smithi195 ceph-mon[49435]: pgmap v287: 97 pgs: 97 active+clean; 332 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 6.9 KiB/s rd, 5.3 MiB/s wr, 461 op/s; 169 KiB/s, 25 keys/s, 5 objects/s recovering 2024-09-21T04:20:54.884 INFO:teuthology.orchestra.run.smithi089.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-21T04:20:54.884 INFO:teuthology.orchestra.run.smithi089.stdout:alertmanager.smithi089 smithi089 *:9093,9094 running (9m) 2m ago 18m 17.9M - 0.27.0 11f11916f8cd 65dfcbcafee1 2024-09-21T04:20:54.884 INFO:teuthology.orchestra.run.smithi089.stdout:crash.smithi089 smithi089 running (5m) 2m ago 18m 6891k - 19.3.0-5103-gf36283e7 48ed0a2cf89d 6c415a5e173e 2024-09-21T04:20:54.884 INFO:teuthology.orchestra.run.smithi089.stdout:crash.smithi195 smithi195 running (5m) 23s ago 17m 6891k - 19.3.0-5103-gf36283e7 48ed0a2cf89d 78ce05a4e9ab 2024-09-21T04:20:54.884 INFO:teuthology.orchestra.run.smithi089.stdout:grafana.smithi089 smithi089 *:3000 running (8m) 2m ago 17m 47.2M - dad864ee21e9 ec4d6738bce7 2024-09-21T04:20:54.884 INFO:teuthology.orchestra.run.smithi089.stdout:mds.foofs.smithi089.osufws smithi089 running (12m) 2m ago 12m 38.5M - 17.2.0 e1d6a67b021e 664d41859bea 2024-09-21T04:20:54.884 INFO:teuthology.orchestra.run.smithi089.stdout:mds.foofs.smithi195.hijxre smithi195 running (12m) 23s ago 12m 17.7M - 17.2.0 e1d6a67b021e 7bc0c6542d12 2024-09-21T04:20:54.884 INFO:teuthology.orchestra.run.smithi089.stdout:mgr.smithi089.srlyxf smithi089 *:8443,9283,8765 running (8m) 2m ago 19m 519M - 19.3.0-5103-gf36283e7 48ed0a2cf89d 3cbeaaa88241 2024-09-21T04:20:54.884 INFO:teuthology.orchestra.run.smithi089.stdout:mgr.smithi195.cgotzm smithi195 *:8443,9283,8765 running (7m) 23s ago 17m 458M - 19.3.0-5103-gf36283e7 48ed0a2cf89d d3734574cf66 2024-09-21T04:20:54.884 INFO:teuthology.orchestra.run.smithi089.stdout:mon.smithi089 smithi089 running (7m) 2m ago 19m 53.6M 2048M 19.3.0-5103-gf36283e7 48ed0a2cf89d 7dd34c8a6f21 2024-09-21T04:20:54.885 INFO:teuthology.orchestra.run.smithi089.stdout:mon.smithi195 smithi195 running (6m) 23s ago 17m 45.6M 2048M 19.3.0-5103-gf36283e7 48ed0a2cf89d 4255937444e7 2024-09-21T04:20:54.885 INFO:teuthology.orchestra.run.smithi089.stdout:nfs.foo.0.0.smithi089.dcjmqc smithi089 *:2049 running (11m) 2m ago 11m 290M - 4.0 e1d6a67b021e 98550f94e773 2024-09-21T04:20:54.885 INFO:teuthology.orchestra.run.smithi089.stdout:nfs.foo.1.0.smithi195.vhoayp smithi195 *:2049 running (11m) 23s ago 11m 88.6M - 4.0 e1d6a67b021e c60a4040130a 2024-09-21T04:20:54.885 INFO:teuthology.orchestra.run.smithi089.stdout:node-exporter.smithi089 smithi089 *:9100 running (8m) 2m ago 17m 9.84M - 1.7.0 72c9c2088986 7ef0d1588475 2024-09-21T04:20:54.885 INFO:teuthology.orchestra.run.smithi089.stdout:node-exporter.smithi195 smithi195 *:9100 running (8m) 23s ago 17m 11.6M - 1.7.0 72c9c2088986 61eafd61bc6c 2024-09-21T04:20:54.885 INFO:teuthology.orchestra.run.smithi089.stdout:osd.0 smithi089 running (5m) 2m ago 16m 331M 1935M 19.3.0-5103-gf36283e7 48ed0a2cf89d 3df0adea59fb 2024-09-21T04:20:54.885 INFO:teuthology.orchestra.run.smithi089.stdout:osd.1 smithi089 running (4m) 2m ago 16m 265M 1935M 19.3.0-5103-gf36283e7 48ed0a2cf89d da2e49774e7e 2024-09-21T04:20:54.885 INFO:teuthology.orchestra.run.smithi089.stdout:osd.2 smithi089 running (3m) 2m ago 15m 202M 1935M 19.3.0-5103-gf36283e7 48ed0a2cf89d b9f6438dd5b9 2024-09-21T04:20:54.885 INFO:teuthology.orchestra.run.smithi089.stdout:osd.3 smithi089 running (2m) 2m ago 15m 11.5M 1935M 19.3.0-5103-gf36283e7 48ed0a2cf89d d269cd94bd08 2024-09-21T04:20:54.885 INFO:teuthology.orchestra.run.smithi089.stdout:osd.4 smithi195 running (105s) 23s ago 14m 391M 2704M 19.3.0-5103-gf36283e7 48ed0a2cf89d e375f90f95c4 2024-09-21T04:20:54.885 INFO:teuthology.orchestra.run.smithi089.stdout:osd.5 smithi195 running (67s) 23s ago 14m 254M 2704M 19.3.0-5103-gf36283e7 48ed0a2cf89d 41de0beb713d 2024-09-21T04:20:54.885 INFO:teuthology.orchestra.run.smithi089.stdout:osd.6 smithi195 running (27s) 23s ago 14m 12.1M 2704M 19.3.0-5103-gf36283e7 48ed0a2cf89d a41d0c0574ab 2024-09-21T04:20:54.885 INFO:teuthology.orchestra.run.smithi089.stdout:osd.7 smithi195 running (13m) 23s ago 13m 746M 2704M 17.2.0 e1d6a67b021e 0109b3fea188 2024-09-21T04:20:54.885 INFO:teuthology.orchestra.run.smithi089.stdout:prometheus.smithi089 smithi089 *:9095 running (7m) 2m ago 17m 53.3M - 2.51.0 1d3b7f56885b 8c9f309a6459 2024-09-21T04:20:55.359 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:20:55 smithi089 ceph-mon[64076]: from='client.44319 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T04:20:55.359 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:20:55 smithi089 ceph-mon[64076]: from='client.34452 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T04:20:55.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:20:55 smithi195 ceph-mon[49435]: from='client.44319 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T04:20:55.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:20:55 smithi195 ceph-mon[49435]: from='client.34452 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T04:20:55.399 INFO:teuthology.orchestra.run.smithi089.stdout: 5 257131 9.03 MB/sec execute 574 sec latency 56.310 ms 2024-09-21T04:20:55.432 INFO:teuthology.orchestra.run.smithi089.stdout:{ 2024-09-21T04:20:55.432 INFO:teuthology.orchestra.run.smithi089.stdout: "mon": { 2024-09-21T04:20:55.432 INFO:teuthology.orchestra.run.smithi089.stdout: "ceph version 19.3.0-5103-gf36283e7 (f36283e7c91cac1c01cb9a7c4ff21bd697dbd4f2) squid (dev)": 2 2024-09-21T04:20:55.432 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2024-09-21T04:20:55.432 INFO:teuthology.orchestra.run.smithi089.stdout: "mgr": { 2024-09-21T04:20:55.432 INFO:teuthology.orchestra.run.smithi089.stdout: "ceph version 19.3.0-5103-gf36283e7 (f36283e7c91cac1c01cb9a7c4ff21bd697dbd4f2) squid (dev)": 2 2024-09-21T04:20:55.433 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2024-09-21T04:20:55.433 INFO:teuthology.orchestra.run.smithi089.stdout: "osd": { 2024-09-21T04:20:55.433 INFO:teuthology.orchestra.run.smithi089.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 1, 2024-09-21T04:20:55.433 INFO:teuthology.orchestra.run.smithi089.stdout: "ceph version 19.3.0-5103-gf36283e7 (f36283e7c91cac1c01cb9a7c4ff21bd697dbd4f2) squid (dev)": 7 2024-09-21T04:20:55.433 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2024-09-21T04:20:55.433 INFO:teuthology.orchestra.run.smithi089.stdout: "mds": { 2024-09-21T04:20:55.433 INFO:teuthology.orchestra.run.smithi089.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-21T04:20:55.433 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2024-09-21T04:20:55.433 INFO:teuthology.orchestra.run.smithi089.stdout: "overall": { 2024-09-21T04:20:55.433 INFO:teuthology.orchestra.run.smithi089.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3, 2024-09-21T04:20:55.433 INFO:teuthology.orchestra.run.smithi089.stdout: "ceph version 19.3.0-5103-gf36283e7 (f36283e7c91cac1c01cb9a7c4ff21bd697dbd4f2) squid (dev)": 11 2024-09-21T04:20:55.433 INFO:teuthology.orchestra.run.smithi089.stdout: } 2024-09-21T04:20:55.433 INFO:teuthology.orchestra.run.smithi089.stdout:} 2024-09-21T04:20:55.909 INFO:teuthology.orchestra.run.smithi089.stdout:{ 2024-09-21T04:20:55.909 INFO:teuthology.orchestra.run.smithi089.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:2195e5f178be0a9390728e2a14088236d8fe6f82635f5c84418fe37a04457e00", 2024-09-21T04:20:55.909 INFO:teuthology.orchestra.run.smithi089.stdout: "in_progress": true, 2024-09-21T04:20:55.909 INFO:teuthology.orchestra.run.smithi089.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-21T04:20:55.909 INFO:teuthology.orchestra.run.smithi089.stdout: "services_complete": [ 2024-09-21T04:20:55.909 INFO:teuthology.orchestra.run.smithi089.stdout: "mgr", 2024-09-21T04:20:55.910 INFO:teuthology.orchestra.run.smithi089.stdout: "crash", 2024-09-21T04:20:55.910 INFO:teuthology.orchestra.run.smithi089.stdout: "mon" 2024-09-21T04:20:55.910 INFO:teuthology.orchestra.run.smithi089.stdout: ], 2024-09-21T04:20:55.910 INFO:teuthology.orchestra.run.smithi089.stdout: "progress": "13/23 daemons upgraded", 2024-09-21T04:20:55.910 INFO:teuthology.orchestra.run.smithi089.stdout: "message": "Currently upgrading osd daemons", 2024-09-21T04:20:55.910 INFO:teuthology.orchestra.run.smithi089.stdout: "is_paused": false 2024-09-21T04:20:55.910 INFO:teuthology.orchestra.run.smithi089.stdout:} 2024-09-21T04:20:56.376 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:20:56 smithi089 ceph-mon[64076]: from='client.? 172.21.15.89:0/3050600712' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T04:20:56.376 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:20:56 smithi089 ceph-mon[64076]: from='client.34460 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T04:20:56.376 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:20:56 smithi089 ceph-mon[64076]: pgmap v288: 97 pgs: 97 active+clean; 336 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.9 MiB/s wr, 498 op/s; 30 KiB/s, 3 objects/s recovering 2024-09-21T04:20:56.399 INFO:teuthology.orchestra.run.smithi089.stdout: 5 257376 9.03 MB/sec execute 575 sec latency 59.322 ms 2024-09-21T04:20:56.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:20:56 smithi195 ceph-mon[49435]: from='client.? 172.21.15.89:0/3050600712' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T04:20:56.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:20:56 smithi195 ceph-mon[49435]: from='client.34460 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T04:20:56.610 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:20:56 smithi195 ceph-mon[49435]: pgmap v288: 97 pgs: 97 active+clean; 336 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.9 MiB/s wr, 498 op/s; 30 KiB/s, 3 objects/s recovering 2024-09-21T04:20:56.651 INFO:teuthology.orchestra.run.smithi089.stdout:HEALTH_OK 2024-09-21T04:20:56.959 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 21 04:20:56 smithi195 systemd[1]: Stopping Ceph osd.7 for 1b9b79fc-77ce-11ef-baf5-efdc52797490... 2024-09-21T04:20:57.216 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 21 04:20:57 smithi195 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-7[39020]: 2024-09-21T04:20:57.014+0000 7ffa7c0d6700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.7 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-21T04:20:57.216 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 21 04:20:57 smithi195 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-7[39020]: 2024-09-21T04:20:57.014+0000 7ffa7c0d6700 -1 osd.7 152 *** Got signal Terminated *** 2024-09-21T04:20:57.216 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 21 04:20:57 smithi195 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-7[39020]: 2024-09-21T04:20:57.014+0000 7ffa7c0d6700 -1 osd.7 152 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-21T04:20:57.390 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:20:57 smithi089 ceph-mon[64076]: from='client.? 172.21.15.89:0/2496525333' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-21T04:20:57.390 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:20:57 smithi089 ceph-mon[64076]: osd.7 marked itself down and dead 2024-09-21T04:20:57.398 INFO:teuthology.orchestra.run.smithi089.stdout: 5 257635 9.02 MB/sec execute 576 sec latency 82.934 ms 2024-09-21T04:20:57.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:20:57 smithi195 ceph-mon[49435]: from='client.? 172.21.15.89:0/2496525333' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-21T04:20:57.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:20:57 smithi195 ceph-mon[49435]: osd.7 marked itself down and dead 2024-09-21T04:20:58.325 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 21 04:20:58 smithi195 podman[59131]: 2024-09-21 04:20:58.020409369 +0000 UTC m=+1.131947133 container stop 0109b3fea188171608f1adbf915ca9c8c50b25bfd6c722d47ecd586dd23653e2 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-7, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_CLEAN=True, com.redhat.component=centos-stream-container, ceph=True, name=centos-stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.tags=base centos centos-stream, version=8, CEPH_POINT_RELEASE=-17.2.0, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, distribution-scope=public, release=754, architecture=x86_64, maintainer=Guillaume Abrioux , vcs-type=git, GIT_REPO=https://github.com/ceph/ceph-container.git, RELEASE=HEAD, io.k8s.display-name=CentOS Stream 8, build-date=2022-05-03T08:36:31.336870, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, vendor=Red Hat, Inc., com.redhat.license_terms=https://centos.org/legal/licensing-policy/, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.openshift.expose-services=, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_BRANCH=HEAD, io.buildah.version=1.19.8) 2024-09-21T04:20:58.326 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 21 04:20:58 smithi195 podman[59131]: 2024-09-21 04:20:58.046681044 +0000 UTC m=+1.158218798 container died 0109b3fea188171608f1adbf915ca9c8c50b25bfd6c722d47ecd586dd23653e2 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-7, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.k8s.display-name=CentOS Stream 8, vcs-type=git, build-date=2022-05-03T08:36:31.336870, io.buildah.version=1.19.8, name=centos-stream, io.openshift.expose-services=, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, RELEASE=HEAD, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_BRANCH=HEAD, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, version=8, release=754, architecture=x86_64, GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_CLEAN=True, com.redhat.component=centos-stream-container, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.openshift.tags=base centos centos-stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, maintainer=Guillaume Abrioux , com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vendor=Red Hat, Inc., distribution-scope=public, ceph=True, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., CEPH_POINT_RELEASE=-17.2.0) 2024-09-21T04:20:58.399 INFO:teuthology.orchestra.run.smithi089.stdout: 5 257670 9.01 MB/sec execute 577 sec latency 1029.127 ms 2024-09-21T04:20:58.579 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:20:58 smithi089 ceph-mon[64076]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-21T04:20:58.579 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:20:58 smithi089 ceph-mon[64076]: Health check failed: all OSDs are running squid or later but require_osd_release < squid (OSD_UPGRADE_FINISHED) 2024-09-21T04:20:58.579 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:20:58 smithi089 ceph-mon[64076]: osdmap e153: 8 total, 7 up, 8 in 2024-09-21T04:20:58.579 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:20:58 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:20:58.579 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:20:58 smithi089 ceph-mon[64076]: pgmap v290: 97 pgs: 10 peering, 9 stale+active+clean, 78 active+clean; 336 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 7.1 KiB/s rd, 4.2 MiB/s wr, 389 op/s 2024-09-21T04:20:58.610 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:20:58 smithi195 ceph-mon[49435]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-21T04:20:58.610 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:20:58 smithi195 ceph-mon[49435]: Health check failed: all OSDs are running squid or later but require_osd_release < squid (OSD_UPGRADE_FINISHED) 2024-09-21T04:20:58.610 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:20:58 smithi195 ceph-mon[49435]: osdmap e153: 8 total, 7 up, 8 in 2024-09-21T04:20:58.610 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:20:58 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:20:58.610 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:20:58 smithi195 ceph-mon[49435]: pgmap v290: 97 pgs: 10 peering, 9 stale+active+clean, 78 active+clean; 336 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 7.1 KiB/s rd, 4.2 MiB/s wr, 389 op/s 2024-09-21T04:20:58.611 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 21 04:20:58 smithi195 podman[59131]: 2024-09-21 04:20:58.413589083 +0000 UTC m=+1.525126839 container cleanup 0109b3fea188171608f1adbf915ca9c8c50b25bfd6c722d47ecd586dd23653e2 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-7, GIT_CLEAN=True, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, CEPH_POINT_RELEASE=-17.2.0, distribution-scope=public, ceph=True, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, maintainer=Guillaume Abrioux , com.redhat.component=centos-stream-container, version=8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, architecture=x86_64, GIT_BRANCH=HEAD, io.k8s.display-name=CentOS Stream 8, vendor=Red Hat, Inc., name=centos-stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_REPO=https://github.com/ceph/ceph-container.git, RELEASE=HEAD, io.openshift.tags=base centos centos-stream, io.openshift.expose-services=, release=754, io.buildah.version=1.19.8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vcs-type=git, build-date=2022-05-03T08:36:31.336870) 2024-09-21T04:20:58.611 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 21 04:20:58 smithi195 bash[59131]: ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-7 2024-09-21T04:20:59.103 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 21 04:20:58 smithi195 podman[59159]: 2024-09-21 04:20:58.615304117 +0000 UTC m=+0.584696293 container remove 0109b3fea188171608f1adbf915ca9c8c50b25bfd6c722d47ecd586dd23653e2 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-7, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, name=centos-stream, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=-17.2.0, vcs-type=git, RELEASE=HEAD, distribution-scope=public, com.redhat.component=centos-stream-container, GIT_CLEAN=True, io.k8s.display-name=CentOS Stream 8, io.openshift.expose-services=, architecture=x86_64, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_REPO=https://github.com/ceph/ceph-container.git, build-date=2022-05-03T08:36:31.336870, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, release=754, GIT_BRANCH=HEAD, io.openshift.tags=base centos centos-stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.buildah.version=1.19.8, ceph=True, version=8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vendor=Red Hat, Inc.) 2024-09-21T04:20:59.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:20:59 smithi195 ceph-mon[49435]: osdmap e154: 8 total, 7 up, 8 in 2024-09-21T04:20:59.359 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 21 04:20:59 smithi195 podman[59241]: 2024-09-21 04:20:59.005271501 +0000 UTC m=+0.021832178 image pull 48ed0a2cf89d6d0371a2c6185a5869a747265f6abd5ac2a74f00cc32d5628721 quay.ceph.io/ceph-ci/ceph@sha256:2195e5f178be0a9390728e2a14088236d8fe6f82635f5c84418fe37a04457e00 2024-09-21T04:20:59.359 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 21 04:20:59 smithi195 podman[59241]: 2024-09-21 04:20:59.149844471 +0000 UTC m=+0.166405155 container create b7c9f530b98d12f6f85cf8105c221ccb1bda725b8141c901621070a77bb1f06a (image=quay.ceph.io/ceph-ci/ceph@sha256:2195e5f178be0a9390728e2a14088236d8fe6f82635f5c84418fe37a04457e00, name=ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-7-deactivate, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, RELEASE=wip-guits-main-2024-09-20-2310-f36283e, ceph=True, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux ) 2024-09-21T04:20:59.382 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:20:59 smithi089 ceph-mon[64076]: osdmap e154: 8 total, 7 up, 8 in 2024-09-21T04:20:59.399 INFO:teuthology.orchestra.run.smithi089.stdout: 5 257956 9.01 MB/sec execute 578 sec latency 84.853 ms 2024-09-21T04:20:59.859 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 21 04:20:59 smithi195 podman[59241]: 2024-09-21 04:20:59.411919842 +0000 UTC m=+0.428480541 container init b7c9f530b98d12f6f85cf8105c221ccb1bda725b8141c901621070a77bb1f06a (image=quay.ceph.io/ceph-ci/ceph@sha256:2195e5f178be0a9390728e2a14088236d8fe6f82635f5c84418fe37a04457e00, name=ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-7-deactivate, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.license=GPLv2, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, RELEASE=wip-guits-main-2024-09-20-2310-f36283e, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2) 2024-09-21T04:20:59.859 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 21 04:20:59 smithi195 podman[59241]: 2024-09-21 04:20:59.416525727 +0000 UTC m=+0.433086412 container start b7c9f530b98d12f6f85cf8105c221ccb1bda725b8141c901621070a77bb1f06a (image=quay.ceph.io/ceph-ci/ceph@sha256:2195e5f178be0a9390728e2a14088236d8fe6f82635f5c84418fe37a04457e00, name=ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-7-deactivate, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, GIT_CLEAN=True, org.label-schema.vendor=CentOS, RELEASE=wip-guits-main-2024-09-20-2310-f36283e, maintainer=Guillaume Abrioux ) 2024-09-21T04:20:59.859 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 21 04:20:59 smithi195 podman[59241]: 2024-09-21 04:20:59.511699159 +0000 UTC m=+0.528259843 container attach b7c9f530b98d12f6f85cf8105c221ccb1bda725b8141c901621070a77bb1f06a (image=quay.ceph.io/ceph-ci/ceph@sha256:2195e5f178be0a9390728e2a14088236d8fe6f82635f5c84418fe37a04457e00, name=ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-7-deactivate, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.license=GPLv2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-guits-main-2024-09-20-2310-f36283e, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2) 2024-09-21T04:20:59.860 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 21 04:20:59 smithi195 podman[59241]: 2024-09-21 04:20:59.650107344 +0000 UTC m=+0.666668027 container died b7c9f530b98d12f6f85cf8105c221ccb1bda725b8141c901621070a77bb1f06a (image=quay.ceph.io/ceph-ci/ceph@sha256:2195e5f178be0a9390728e2a14088236d8fe6f82635f5c84418fe37a04457e00, name=ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-7-deactivate, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, RELEASE=wip-guits-main-2024-09-20-2310-f36283e, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.build-date=20240731) 2024-09-21T04:21:00.399 INFO:teuthology.orchestra.run.smithi089.stdout: 5 258353 9.01 MB/sec execute 579 sec latency 39.091 ms 2024-09-21T04:21:00.405 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:21:00 smithi089 ceph-mon[64076]: pgmap v292: 97 pgs: 3 active+undersized, 18 peering, 15 active+undersized+degraded, 61 active+clean; 328 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 7.4 KiB/s rd, 5.4 MiB/s wr, 495 op/s; 126/1515 objects degraded (8.317%) 2024-09-21T04:21:00.435 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:21:00 smithi195 ceph-mon[49435]: pgmap v292: 97 pgs: 3 active+undersized, 18 peering, 15 active+undersized+degraded, 61 active+clean; 328 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 7.4 KiB/s rd, 5.4 MiB/s wr, 495 op/s; 126/1515 objects degraded (8.317%) 2024-09-21T04:21:00.784 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 21 04:21:00 smithi195 podman[59264]: 2024-09-21 04:21:00.433515472 +0000 UTC m=+0.772476582 container remove b7c9f530b98d12f6f85cf8105c221ccb1bda725b8141c901621070a77bb1f06a (image=quay.ceph.io/ceph-ci/ceph@sha256:2195e5f178be0a9390728e2a14088236d8fe6f82635f5c84418fe37a04457e00, name=ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-7-deactivate, RELEASE=wip-guits-main-2024-09-20-2310-f36283e, GIT_BRANCH=HEAD, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731) 2024-09-21T04:21:00.784 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 21 04:21:00 smithi195 systemd[1]: ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490@osd.7.service: Deactivated successfully. 2024-09-21T04:21:00.785 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 21 04:21:00 smithi195 systemd[1]: Stopped Ceph osd.7 for 1b9b79fc-77ce-11ef-baf5-efdc52797490. 2024-09-21T04:21:00.785 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 21 04:21:00 smithi195 systemd[1]: ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490@osd.7.service: Consumed 6min 3.952s CPU time. 2024-09-21T04:21:01.087 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 21 04:21:00 smithi195 systemd[1]: Starting Ceph osd.7 for 1b9b79fc-77ce-11ef-baf5-efdc52797490... 2024-09-21T04:21:01.087 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 21 04:21:01 smithi195 podman[59361]: 2024-09-21 04:21:00.925934918 +0000 UTC m=+0.019682930 image pull 48ed0a2cf89d6d0371a2c6185a5869a747265f6abd5ac2a74f00cc32d5628721 quay.ceph.io/ceph-ci/ceph@sha256:2195e5f178be0a9390728e2a14088236d8fe6f82635f5c84418fe37a04457e00 2024-09-21T04:21:01.087 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 21 04:21:01 smithi195 podman[59361]: 2024-09-21 04:21:01.084802426 +0000 UTC m=+0.178550421 container create b144d5028e57b89d48afa8ba4e4a5bec6549281bb78863c1a955af598d4573b1 (image=quay.ceph.io/ceph-ci/ceph@sha256:2195e5f178be0a9390728e2a14088236d8fe6f82635f5c84418fe37a04457e00, name=ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-7-activate, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.license=GPLv2, RELEASE=wip-guits-main-2024-09-20-2310-f36283e, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731) 2024-09-21T04:21:01.399 INFO:teuthology.orchestra.run.smithi089.stdout: 5 258661 9.00 MB/sec execute 580 sec latency 59.997 ms 2024-09-21T04:21:01.424 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:21:01 smithi089 ceph-mon[64076]: Health check failed: Degraded data redundancy: 126/1515 objects degraded (8.317%), 15 pgs degraded (PG_DEGRADED) 2024-09-21T04:21:01.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:21:01 smithi195 ceph-mon[49435]: Health check failed: Degraded data redundancy: 126/1515 objects degraded (8.317%), 15 pgs degraded (PG_DEGRADED) 2024-09-21T04:21:01.610 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 21 04:21:01 smithi195 podman[59361]: 2024-09-21 04:21:01.3380944 +0000 UTC m=+0.431842396 container init b144d5028e57b89d48afa8ba4e4a5bec6549281bb78863c1a955af598d4573b1 (image=quay.ceph.io/ceph-ci/ceph@sha256:2195e5f178be0a9390728e2a14088236d8fe6f82635f5c84418fe37a04457e00, name=ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-7-activate, io.buildah.version=1.37.2, ceph=True, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, RELEASE=wip-guits-main-2024-09-20-2310-f36283e, org.label-schema.vendor=CentOS, GIT_CLEAN=True) 2024-09-21T04:21:01.610 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 21 04:21:01 smithi195 podman[59361]: 2024-09-21 04:21:01.343644003 +0000 UTC m=+0.437392000 container start b144d5028e57b89d48afa8ba4e4a5bec6549281bb78863c1a955af598d4573b1 (image=quay.ceph.io/ceph-ci/ceph@sha256:2195e5f178be0a9390728e2a14088236d8fe6f82635f5c84418fe37a04457e00, name=ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-7-activate, RELEASE=wip-guits-main-2024-09-20-2310-f36283e, GIT_CLEAN=True, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23) 2024-09-21T04:21:01.610 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 21 04:21:01 smithi195 podman[59361]: 2024-09-21 04:21:01.446633852 +0000 UTC m=+0.540381857 container attach b144d5028e57b89d48afa8ba4e4a5bec6549281bb78863c1a955af598d4573b1 (image=quay.ceph.io/ceph-ci/ceph@sha256:2195e5f178be0a9390728e2a14088236d8fe6f82635f5c84418fe37a04457e00, name=ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-7-activate, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, RELEASE=wip-guits-main-2024-09-20-2310-f36283e, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, GIT_CLEAN=True, ceph=True, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23) 2024-09-21T04:21:01.610 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 21 04:21:01 smithi195 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-7-activate[59375]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T04:21:01.610 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 21 04:21:01 smithi195 bash[59361]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T04:21:01.610 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 21 04:21:01 smithi195 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-7-activate[59375]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T04:21:01.610 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 21 04:21:01 smithi195 bash[59361]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T04:21:02.399 INFO:teuthology.orchestra.run.smithi089.stdout: 5 259258 9.02 MB/sec execute 581 sec latency 30.449 ms 2024-09-21T04:21:02.435 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:21:02 smithi089 ceph-mon[64076]: pgmap v293: 97 pgs: 3 active+undersized, 18 peering, 15 active+undersized+degraded, 61 active+clean; 328 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 3.0 MiB/s wr, 372 op/s; 126/1506 objects degraded (8.367%) 2024-09-21T04:21:02.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:21:02 smithi195 ceph-mon[49435]: pgmap v293: 97 pgs: 3 active+undersized, 18 peering, 15 active+undersized+degraded, 61 active+clean; 328 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 3.0 MiB/s wr, 372 op/s; 126/1506 objects degraded (8.367%) 2024-09-21T04:21:03.399 INFO:teuthology.orchestra.run.smithi089.stdout: 5 259690 9.03 MB/sec execute 582 sec latency 37.389 ms 2024-09-21T04:21:03.462 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:21:03 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:21:03.462 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:21:03 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T04:21:03.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:21:03 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:21:03.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:21:03 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T04:21:04.399 INFO:teuthology.orchestra.run.smithi089.stdout: 5 260113 9.04 MB/sec execute 583 sec latency 56.537 ms 2024-09-21T04:21:04.484 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:21:04 smithi089 ceph-mon[64076]: pgmap v294: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 336 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 6.4 KiB/s rd, 4.7 MiB/s wr, 532 op/s; 218/1650 objects degraded (13.212%) 2024-09-21T04:21:04.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:21:04 smithi195 ceph-mon[49435]: pgmap v294: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 336 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 6.4 KiB/s rd, 4.7 MiB/s wr, 532 op/s; 218/1650 objects degraded (13.212%) 2024-09-21T04:21:04.609 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 21 04:21:04 smithi195 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-7-activate[59375]: --> Failed to activate via raw: did not find any matching OSD to activate 2024-09-21T04:21:04.609 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 21 04:21:04 smithi195 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-7-activate[59375]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T04:21:04.609 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 21 04:21:04 smithi195 bash[59361]: --> Failed to activate via raw: did not find any matching OSD to activate 2024-09-21T04:21:04.610 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 21 04:21:04 smithi195 bash[59361]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T04:21:04.610 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 21 04:21:04 smithi195 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-7-activate[59375]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T04:21:04.610 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 21 04:21:04 smithi195 bash[59361]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T04:21:04.610 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 21 04:21:04 smithi195 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-7-activate[59375]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-7 2024-09-21T04:21:04.610 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 21 04:21:04 smithi195 bash[59361]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-7 2024-09-21T04:21:04.610 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 21 04:21:04 smithi195 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-7-activate[59375]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_1 --path /var/lib/ceph/osd/ceph-7 --no-mon-config 2024-09-21T04:21:04.610 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 21 04:21:04 smithi195 bash[59361]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_1 --path /var/lib/ceph/osd/ceph-7 --no-mon-config 2024-09-21T04:21:05.109 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 21 04:21:04 smithi195 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-7-activate[59375]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_1 /var/lib/ceph/osd/ceph-7/block 2024-09-21T04:21:05.109 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 21 04:21:04 smithi195 bash[59361]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_1 /var/lib/ceph/osd/ceph-7/block 2024-09-21T04:21:05.109 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 21 04:21:04 smithi195 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-7-activate[59375]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-7/block 2024-09-21T04:21:05.110 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 21 04:21:04 smithi195 bash[59361]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-7/block 2024-09-21T04:21:05.110 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 21 04:21:04 smithi195 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-7-activate[59375]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-09-21T04:21:05.110 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 21 04:21:04 smithi195 bash[59361]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-09-21T04:21:05.110 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 21 04:21:04 smithi195 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-7-activate[59375]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-7 2024-09-21T04:21:05.110 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 21 04:21:04 smithi195 bash[59361]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-7 2024-09-21T04:21:05.110 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 21 04:21:04 smithi195 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-7-activate[59375]: --> ceph-volume lvm activate successful for osd ID: 7 2024-09-21T04:21:05.110 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 21 04:21:04 smithi195 bash[59361]: --> ceph-volume lvm activate successful for osd ID: 7 2024-09-21T04:21:05.110 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 21 04:21:04 smithi195 conmon[59375]: conmon b144d5028e57b89d48af : Failed to open cgroups file: /sys/fs/cgroup/machine.slice/libpod-b144d5028e57b89d48afa8ba4e4a5bec6549281bb78863c1a955af598d4573b1.scope/container/memory.events 2024-09-21T04:21:05.110 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 21 04:21:04 smithi195 podman[59590]: 2024-09-21 04:21:04.836863767 +0000 UTC m=+0.020954069 container died b144d5028e57b89d48afa8ba4e4a5bec6549281bb78863c1a955af598d4573b1 (image=quay.ceph.io/ceph-ci/ceph@sha256:2195e5f178be0a9390728e2a14088236d8fe6f82635f5c84418fe37a04457e00, name=ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-7-activate, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, RELEASE=wip-guits-main-2024-09-20-2310-f36283e, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-21T04:21:05.399 INFO:teuthology.orchestra.run.smithi089.stdout: 5 260514 9.04 MB/sec execute 584 sec latency 38.292 ms 2024-09-21T04:21:06.255 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:21:06 smithi089 ceph-mon[64076]: Health check update: Degraded data redundancy: 231/1698 objects degraded (13.604%), 29 pgs degraded (PG_DEGRADED) 2024-09-21T04:21:06.399 INFO:teuthology.orchestra.run.smithi089.stdout: 5 260794 9.03 MB/sec execute 585 sec latency 40.888 ms 2024-09-21T04:21:06.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:21:06 smithi195 ceph-mon[49435]: Health check update: Degraded data redundancy: 231/1698 objects degraded (13.604%), 29 pgs degraded (PG_DEGRADED) 2024-09-21T04:21:07.081 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 21 04:21:06 smithi195 podman[59590]: 2024-09-21 04:21:06.827509413 +0000 UTC m=+2.011599704 container remove b144d5028e57b89d48afa8ba4e4a5bec6549281bb78863c1a955af598d4573b1 (image=quay.ceph.io/ceph-ci/ceph@sha256:2195e5f178be0a9390728e2a14088236d8fe6f82635f5c84418fe37a04457e00, name=ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-7-activate, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, ceph=True, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, RELEASE=wip-guits-main-2024-09-20-2310-f36283e, maintainer=Guillaume Abrioux ) 2024-09-21T04:21:07.081 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 21 04:21:07 smithi195 podman[59649]: 2024-09-21 04:21:06.946845754 +0000 UTC m=+0.017105404 image pull 48ed0a2cf89d6d0371a2c6185a5869a747265f6abd5ac2a74f00cc32d5628721 quay.ceph.io/ceph-ci/ceph@sha256:2195e5f178be0a9390728e2a14088236d8fe6f82635f5c84418fe37a04457e00 2024-09-21T04:21:07.081 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 21 04:21:07 smithi195 podman[59649]: 2024-09-21 04:21:07.07894117 +0000 UTC m=+0.149200802 container create 66d3bf5f6c5fb57028aaee93e64eaeea1aee46159f007c42971b16fec7fe10e9 (image=quay.ceph.io/ceph-ci/ceph@sha256:2195e5f178be0a9390728e2a14088236d8fe6f82635f5c84418fe37a04457e00, name=ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-7, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_CLEAN=True, maintainer=Guillaume Abrioux , RELEASE=wip-guits-main-2024-09-20-2310-f36283e, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.build-date=20240731, ceph=True) 2024-09-21T04:21:07.274 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:21:07 smithi089 ceph-mon[64076]: pgmap v295: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 334 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 5.9 KiB/s rd, 5.2 MiB/s wr, 521 op/s; 231/1698 objects degraded (13.604%) 2024-09-21T04:21:07.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:21:07 smithi195 ceph-mon[49435]: pgmap v295: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 334 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 5.9 KiB/s rd, 5.2 MiB/s wr, 521 op/s; 231/1698 objects degraded (13.604%) 2024-09-21T04:21:07.399 INFO:teuthology.orchestra.run.smithi089.stdout: 5 261257 9.04 MB/sec execute 586 sec latency 47.318 ms 2024-09-21T04:21:07.609 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 21 04:21:07 smithi195 podman[59649]: 2024-09-21 04:21:07.357046912 +0000 UTC m=+0.427306540 container init 66d3bf5f6c5fb57028aaee93e64eaeea1aee46159f007c42971b16fec7fe10e9 (image=quay.ceph.io/ceph-ci/ceph@sha256:2195e5f178be0a9390728e2a14088236d8fe6f82635f5c84418fe37a04457e00, name=ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-7, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, RELEASE=wip-guits-main-2024-09-20-2310-f36283e, org.label-schema.license=GPLv2, ceph=True, maintainer=Guillaume Abrioux , GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS) 2024-09-21T04:21:07.609 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 21 04:21:07 smithi195 podman[59649]: 2024-09-21 04:21:07.361066426 +0000 UTC m=+0.431326061 container start 66d3bf5f6c5fb57028aaee93e64eaeea1aee46159f007c42971b16fec7fe10e9 (image=quay.ceph.io/ceph-ci/ceph@sha256:2195e5f178be0a9390728e2a14088236d8fe6f82635f5c84418fe37a04457e00, name=ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-7, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, ceph=True, RELEASE=wip-guits-main-2024-09-20-2310-f36283e, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731) 2024-09-21T04:21:07.609 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 21 04:21:07 smithi195 ceph-osd[59668]: -- 172.21.15.195:0/823358188 <== mon.1 v2:172.21.15.195:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55f748b781e0 con 0x55f747d13c00 2024-09-21T04:21:07.609 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 21 04:21:07 smithi195 bash[59649]: 66d3bf5f6c5fb57028aaee93e64eaeea1aee46159f007c42971b16fec7fe10e9 2024-09-21T04:21:07.609 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 21 04:21:07 smithi195 systemd[1]: Started Ceph osd.7 for 1b9b79fc-77ce-11ef-baf5-efdc52797490. 2024-09-21T04:21:08.399 INFO:teuthology.orchestra.run.smithi089.stdout: 5 261638 9.04 MB/sec execute 587 sec latency 50.728 ms 2024-09-21T04:21:08.538 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:21:08 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:21:08.538 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:21:08 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:21:08.538 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:21:08 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T04:21:08.538 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:21:08 smithi089 ceph-mon[64076]: pgmap v296: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 333 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 5.7 KiB/s rd, 5.0 MiB/s wr, 503 op/s; 238/1734 objects degraded (13.725%) 2024-09-21T04:21:08.859 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:21:08 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:21:08.859 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:21:08 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:21:08.859 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:21:08 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T04:21:08.859 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:21:08 smithi195 ceph-mon[49435]: pgmap v296: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 333 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 5.7 KiB/s rd, 5.0 MiB/s wr, 503 op/s; 238/1734 objects degraded (13.725%) 2024-09-21T04:21:09.293 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 21 04:21:09 smithi195 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-7[59664]: 2024-09-21T04:21:09.256+0000 7f99afe97640 -1 Falling back to public interface 2024-09-21T04:21:09.399 INFO:teuthology.orchestra.run.smithi089.stdout: 5 261961 9.03 MB/sec execute 588 sec latency 40.823 ms 2024-09-21T04:21:10.399 INFO:teuthology.orchestra.run.smithi089.stdout: 5 262345 9.04 MB/sec execute 589 sec latency 39.332 ms 2024-09-21T04:21:11.338 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:21:11 smithi089 ceph-mon[64076]: pgmap v297: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 331 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 4.9 KiB/s rd, 5.3 MiB/s wr, 492 op/s; 231/1680 objects degraded (13.750%) 2024-09-21T04:21:11.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:21:11 smithi195 ceph-mon[49435]: pgmap v297: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 331 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 4.9 KiB/s rd, 5.3 MiB/s wr, 492 op/s; 231/1680 objects degraded (13.750%) 2024-09-21T04:21:11.399 INFO:teuthology.orchestra.run.smithi089.stdout: 5 262683 9.04 MB/sec execute 590 sec latency 68.992 ms 2024-09-21T04:21:12.399 INFO:teuthology.orchestra.run.smithi089.stdout: 5 263094 9.05 MB/sec execute 591 sec latency 43.108 ms 2024-09-21T04:21:12.613 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:21:12 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:21:12.613 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:21:12 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:21:12.613 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:21:12 smithi089 ceph-mon[64076]: pgmap v298: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 331 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 5.7 KiB/s rd, 4.6 MiB/s wr, 454 op/s; 231/1689 objects degraded (13.677%) 2024-09-21T04:21:12.613 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:21:12 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:21:12.613 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:21:12 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:21:12.695 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:21:12 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:21:12.696 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:21:12 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:21:12.696 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:21:12 smithi195 ceph-mon[49435]: pgmap v298: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 331 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 5.7 KiB/s rd, 4.6 MiB/s wr, 454 op/s; 231/1689 objects degraded (13.677%) 2024-09-21T04:21:12.696 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:21:12 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:21:12.696 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:21:12 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:21:13.399 INFO:teuthology.orchestra.run.smithi089.stdout: 5 263521 9.06 MB/sec execute 592 sec latency 48.342 ms 2024-09-21T04:21:13.633 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:21:13 smithi089 ceph-mon[64076]: Health check update: Degraded data redundancy: 231/1689 objects degraded (13.677%), 29 pgs degraded (PG_DEGRADED) 2024-09-21T04:21:13.787 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:21:13 smithi195 ceph-mon[49435]: Health check update: Degraded data redundancy: 231/1689 objects degraded (13.677%), 29 pgs degraded (PG_DEGRADED) 2024-09-21T04:21:14.399 INFO:teuthology.orchestra.run.smithi089.stdout: 5 263953 9.06 MB/sec execute 593 sec latency 54.979 ms 2024-09-21T04:21:14.648 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:21:14 smithi089 ceph-mon[64076]: pgmap v299: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 330 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 9.0 KiB/s rd, 6.4 MiB/s wr, 666 op/s; 232/1629 objects degraded (14.242%) 2024-09-21T04:21:14.835 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:21:14 smithi195 ceph-mon[49435]: pgmap v299: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 330 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 9.0 KiB/s rd, 6.4 MiB/s wr, 666 op/s; 232/1629 objects degraded (14.242%) 2024-09-21T04:21:15.399 INFO:teuthology.orchestra.run.smithi089.stdout: 5 264255 9.05 MB/sec execute 594 sec latency 42.670 ms 2024-09-21T04:21:16.399 INFO:teuthology.orchestra.run.smithi089.stdout: 5 264607 9.05 MB/sec execute 595 sec latency 33.655 ms 2024-09-21T04:21:16.744 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 21 04:21:16 smithi195 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-7[59664]: 2024-09-21T04:21:16.482+0000 7f99afe97640 -1 osd.7 0 read_superblock omap replica is missing. 2024-09-21T04:21:16.933 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:21:16 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:21:16.933 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:21:16 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:21:16.934 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:21:16 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T04:21:16.934 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:21:16 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T04:21:16.934 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:21:16 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:21:16.934 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:21:16 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:21:16.934 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:21:16 smithi089 ceph-mon[64076]: pgmap v300: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 335 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 6.8 KiB/s rd, 5.6 MiB/s wr, 526 op/s; 239/1677 objects degraded (14.252%) 2024-09-21T04:21:16.934 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:21:16 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T04:21:16.934 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:21:16 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "versions"} : dispatch 2024-09-21T04:21:16.934 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:21:16 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "versions"} : dispatch 2024-09-21T04:21:16.935 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:21:16 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "versions"} : dispatch 2024-09-21T04:21:16.935 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:21:16 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "versions"} : dispatch 2024-09-21T04:21:16.935 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:21:16 smithi089 ceph-mon[64076]: Upgrade: Setting container_image for all osd 2024-09-21T04:21:16.935 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:21:16 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:21:16.935 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:21:16 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.0"} : dispatch 2024-09-21T04:21:16.935 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:21:16 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.0"}]': finished 2024-09-21T04:21:16.935 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:21:16 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.1"} : dispatch 2024-09-21T04:21:16.935 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:21:16 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.1"}]': finished 2024-09-21T04:21:16.935 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:21:16 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.2"} : dispatch 2024-09-21T04:21:16.935 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:21:16 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.2"}]': finished 2024-09-21T04:21:16.935 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:21:16 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.3"} : dispatch 2024-09-21T04:21:16.935 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:21:16 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.3"}]': finished 2024-09-21T04:21:16.935 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:21:16 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.4"} : dispatch 2024-09-21T04:21:16.936 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:21:16 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.4"}]': finished 2024-09-21T04:21:16.936 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:21:16 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.5"} : dispatch 2024-09-21T04:21:16.936 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:21:16 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.5"}]': finished 2024-09-21T04:21:16.936 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:21:16 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.6"} : dispatch 2024-09-21T04:21:16.936 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:21:16 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.6"}]': finished 2024-09-21T04:21:16.936 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:21:16 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.7"} : dispatch 2024-09-21T04:21:16.936 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:21:16 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.7"}]': finished 2024-09-21T04:21:16.936 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:21:16 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "osd require-osd-release", "release": "squid"} : dispatch 2024-09-21T04:21:17.110 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:21:16 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:21:17.110 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:21:16 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:21:17.110 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:21:16 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T04:21:17.110 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:21:16 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T04:21:17.110 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:21:16 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:21:17.110 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:21:16 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:21:17.110 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:21:16 smithi195 ceph-mon[49435]: pgmap v300: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 335 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 6.8 KiB/s rd, 5.6 MiB/s wr, 526 op/s; 239/1677 objects degraded (14.252%) 2024-09-21T04:21:17.110 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:21:16 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T04:21:17.111 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:21:16 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "versions"} : dispatch 2024-09-21T04:21:17.111 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:21:16 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "versions"} : dispatch 2024-09-21T04:21:17.111 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:21:16 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "versions"} : dispatch 2024-09-21T04:21:17.111 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:21:16 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "versions"} : dispatch 2024-09-21T04:21:17.111 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:21:16 smithi195 ceph-mon[49435]: Upgrade: Setting container_image for all osd 2024-09-21T04:21:17.111 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:21:16 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:21:17.111 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:21:16 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.0"} : dispatch 2024-09-21T04:21:17.111 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:21:16 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.0"}]': finished 2024-09-21T04:21:17.111 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:21:16 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.1"} : dispatch 2024-09-21T04:21:17.112 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:21:16 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.1"}]': finished 2024-09-21T04:21:17.112 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:21:16 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.2"} : dispatch 2024-09-21T04:21:17.112 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:21:16 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.2"}]': finished 2024-09-21T04:21:17.112 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:21:16 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.3"} : dispatch 2024-09-21T04:21:17.112 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:21:16 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.3"}]': finished 2024-09-21T04:21:17.112 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:21:16 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.4"} : dispatch 2024-09-21T04:21:17.112 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:21:16 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.4"}]': finished 2024-09-21T04:21:17.112 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:21:16 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.5"} : dispatch 2024-09-21T04:21:17.112 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:21:16 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.5"}]': finished 2024-09-21T04:21:17.112 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:21:16 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.6"} : dispatch 2024-09-21T04:21:17.112 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:21:16 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.6"}]': finished 2024-09-21T04:21:17.113 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:21:16 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.7"} : dispatch 2024-09-21T04:21:17.113 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:21:16 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.7"}]': finished 2024-09-21T04:21:17.113 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:21:16 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "osd require-osd-release", "release": "squid"} : dispatch 2024-09-21T04:21:17.399 INFO:teuthology.orchestra.run.smithi089.stdout: 5 265015 9.05 MB/sec execute 596 sec latency 47.269 ms 2024-09-21T04:21:17.945 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:21:17 smithi089 ceph-mon[64076]: Upgrade: Setting require_osd_release to 19 squid 2024-09-21T04:21:17.945 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:21:17 smithi089 ceph-mon[64076]: Health check cleared: OSD_UPGRADE_FINISHED (was: all OSDs are running squid or later but require_osd_release < squid) 2024-09-21T04:21:17.945 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:21:17 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd='[{"prefix": "osd require-osd-release", "release": "squid"}]': finished 2024-09-21T04:21:17.945 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:21:17 smithi089 ceph-mon[64076]: osdmap e155: 8 total, 7 up, 8 in 2024-09-21T04:21:17.945 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:21:17 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "mds ok-to-stop", "ids": ["foofs.smithi089.osufws"]} : dispatch 2024-09-21T04:21:17.945 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:21:17 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:21:17.945 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:21:17 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T04:21:17.945 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:21:17 smithi089 ceph-mon[64076]: from='osd.7 [v2:172.21.15.195:6824/3037504167,v1:172.21.15.195:6825/3037504167]' entity='osd.7' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]} : dispatch 2024-09-21T04:21:17.945 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:21:17 smithi089 ceph-mon[64076]: from='osd.7 ' entity='osd.7' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]} : dispatch 2024-09-21T04:21:18.109 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 21 04:21:17 smithi195 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-7[59664]: 2024-09-21T04:21:17.724+0000 7f99afe97640 -1 osd.7 152 log_to_monitors true 2024-09-21T04:21:18.109 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:21:17 smithi195 ceph-mon[49435]: Upgrade: Setting require_osd_release to 19 squid 2024-09-21T04:21:18.109 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:21:17 smithi195 ceph-mon[49435]: Health check cleared: OSD_UPGRADE_FINISHED (was: all OSDs are running squid or later but require_osd_release < squid) 2024-09-21T04:21:18.109 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:21:17 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd='[{"prefix": "osd require-osd-release", "release": "squid"}]': finished 2024-09-21T04:21:18.109 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:21:17 smithi195 ceph-mon[49435]: osdmap e155: 8 total, 7 up, 8 in 2024-09-21T04:21:18.109 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:21:17 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "mds ok-to-stop", "ids": ["foofs.smithi089.osufws"]} : dispatch 2024-09-21T04:21:18.110 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:21:17 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:21:18.110 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:21:17 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T04:21:18.110 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:21:17 smithi195 ceph-mon[49435]: from='osd.7 [v2:172.21.15.195:6824/3037504167,v1:172.21.15.195:6825/3037504167]' entity='osd.7' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]} : dispatch 2024-09-21T04:21:18.110 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:21:17 smithi195 ceph-mon[49435]: from='osd.7 ' entity='osd.7' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]} : dispatch 2024-09-21T04:21:18.399 INFO:teuthology.orchestra.run.smithi089.stdout: 5 265418 9.06 MB/sec execute 597 sec latency 52.554 ms 2024-09-21T04:21:18.973 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:21:18 smithi089 ceph-mon[64076]: Upgrade: It appears safe to stop mds.foofs.smithi089.osufws 2024-09-21T04:21:18.973 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:21:18 smithi089 ceph-mon[64076]: pgmap v302: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 331 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 8.3 KiB/s rd, 5.7 MiB/s wr, 567 op/s; 236/1683 objects degraded (14.023%) 2024-09-21T04:21:18.973 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:21:18 smithi089 ceph-mon[64076]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-09-21T04:21:18.973 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:21:18 smithi089 ceph-mon[64076]: osdmap e156: 8 total, 7 up, 8 in 2024-09-21T04:21:18.973 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:21:18 smithi089 ceph-mon[64076]: from='osd.7 [v2:172.21.15.195:6824/3037504167,v1:172.21.15.195:6825/3037504167]' entity='osd.7' cmd={"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi195", "root=default"]} : dispatch 2024-09-21T04:21:18.973 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:21:18 smithi089 ceph-mon[64076]: from='osd.7 ' entity='osd.7' cmd={"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi195", "root=default"]} : dispatch 2024-09-21T04:21:18.973 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:21:18 smithi089 ceph-mon[64076]: from='osd.7 ' entity='osd.7' 2024-09-21T04:21:19.109 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 21 04:21:18 smithi195 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-7[59664]: 2024-09-21T04:21:18.627+0000 7f99a3c39640 -1 osd.7 152 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-21T04:21:19.109 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:21:18 smithi195 ceph-mon[49435]: Upgrade: It appears safe to stop mds.foofs.smithi089.osufws 2024-09-21T04:21:19.109 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:21:18 smithi195 ceph-mon[49435]: pgmap v302: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 331 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 8.3 KiB/s rd, 5.7 MiB/s wr, 567 op/s; 236/1683 objects degraded (14.023%) 2024-09-21T04:21:19.109 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:21:18 smithi195 ceph-mon[49435]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-09-21T04:21:19.109 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:21:18 smithi195 ceph-mon[49435]: osdmap e156: 8 total, 7 up, 8 in 2024-09-21T04:21:19.109 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:21:18 smithi195 ceph-mon[49435]: from='osd.7 [v2:172.21.15.195:6824/3037504167,v1:172.21.15.195:6825/3037504167]' entity='osd.7' cmd={"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi195", "root=default"]} : dispatch 2024-09-21T04:21:19.110 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:21:18 smithi195 ceph-mon[49435]: from='osd.7 ' entity='osd.7' cmd={"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi195", "root=default"]} : dispatch 2024-09-21T04:21:19.110 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:21:18 smithi195 ceph-mon[49435]: from='osd.7 ' entity='osd.7' 2024-09-21T04:21:19.399 INFO:teuthology.orchestra.run.smithi089.stdout: 5 265778 9.05 MB/sec execute 598 sec latency 56.541 ms 2024-09-21T04:21:20.109 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:21:19 smithi195 ceph-mon[49435]: Health check update: Degraded data redundancy: 236/1683 objects degraded (14.023%), 29 pgs degraded (PG_DEGRADED) 2024-09-21T04:21:20.109 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:21:19 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:21:20.109 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:21:19 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "auth get-or-create", "entity": "mds.foofs.smithi089.osufws", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-09-21T04:21:20.109 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:21:19 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T04:21:20.109 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:21:19 smithi195 ceph-mon[49435]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-21T04:21:20.110 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:21:19 smithi195 ceph-mon[49435]: osd.7 [v2:172.21.15.195:6824/3037504167,v1:172.21.15.195:6825/3037504167] boot 2024-09-21T04:21:20.110 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:21:19 smithi195 ceph-mon[49435]: osdmap e157: 8 total, 8 up, 8 in 2024-09-21T04:21:20.110 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:21:19 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "osd metadata", "id": 7} : dispatch 2024-09-21T04:21:20.195 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:21:19 smithi089 ceph-mon[64076]: Health check update: Degraded data redundancy: 236/1683 objects degraded (14.023%), 29 pgs degraded (PG_DEGRADED) 2024-09-21T04:21:20.195 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:21:19 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:21:20.195 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:21:19 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "auth get-or-create", "entity": "mds.foofs.smithi089.osufws", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-09-21T04:21:20.195 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:21:19 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T04:21:20.195 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:21:19 smithi089 ceph-mon[64076]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-21T04:21:20.196 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:21:19 smithi089 ceph-mon[64076]: osd.7 [v2:172.21.15.195:6824/3037504167,v1:172.21.15.195:6825/3037504167] boot 2024-09-21T04:21:20.196 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:21:19 smithi089 ceph-mon[64076]: osdmap e157: 8 total, 8 up, 8 in 2024-09-21T04:21:20.196 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:21:19 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "osd metadata", "id": 7} : dispatch 2024-09-21T04:21:20.399 INFO:teuthology.orchestra.run.smithi089.stdout: 5 265937 9.05 MB/sec execute 599 sec latency 777.527 ms 2024-09-21T04:21:21.015 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:21:20 smithi089 ceph-mon[64076]: Upgrade: Updating mds.foofs.smithi089.osufws 2024-09-21T04:21:21.015 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:21:20 smithi089 ceph-mon[64076]: Deploying daemon mds.foofs.smithi089.osufws on smithi089 2024-09-21T04:21:21.015 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:21:20 smithi089 ceph-mon[64076]: osdmap e158: 8 total, 8 up, 8 in 2024-09-21T04:21:21.015 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:21:20 smithi089 ceph-mon[64076]: pgmap v306: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 331 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 5.7 MiB/s wr, 607 op/s; 223/1656 objects degraded (13.466%) 2024-09-21T04:21:21.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:21:20 smithi195 ceph-mon[49435]: Upgrade: Updating mds.foofs.smithi089.osufws 2024-09-21T04:21:21.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:21:20 smithi195 ceph-mon[49435]: Deploying daemon mds.foofs.smithi089.osufws on smithi089 2024-09-21T04:21:21.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:21:20 smithi195 ceph-mon[49435]: osdmap e158: 8 total, 8 up, 8 in 2024-09-21T04:21:21.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:21:20 smithi195 ceph-mon[49435]: pgmap v306: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 331 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 5.7 MiB/s wr, 607 op/s; 223/1656 objects degraded (13.466%) 2024-09-21T04:21:21.399 INFO:teuthology.orchestra.run.smithi089.stdout: 5 cleanup 600 sec 2024-09-21T04:21:21.683 INFO:teuthology.orchestra.run.smithi089.stdout: 0 cleanup 600 sec 2024-09-21T04:21:21.683 INFO:teuthology.orchestra.run.smithi089.stdout: 2024-09-21T04:21:21.683 INFO:teuthology.orchestra.run.smithi089.stdout: Operation Count AvgLat MaxLat 2024-09-21T04:21:21.683 INFO:teuthology.orchestra.run.smithi089.stdout: ---------------------------------------- 2024-09-21T04:21:21.683 INFO:teuthology.orchestra.run.smithi089.stdout: NTCreateX 174019 4.853 2206.948 2024-09-21T04:21:21.683 INFO:teuthology.orchestra.run.smithi089.stdout: Close 127895 0.132 18.004 2024-09-21T04:21:21.683 INFO:teuthology.orchestra.run.smithi089.stdout: Rename 7374 10.117 2070.406 2024-09-21T04:21:21.683 INFO:teuthology.orchestra.run.smithi089.stdout: Unlink 35038 7.289 1615.004 2024-09-21T04:21:21.683 INFO:teuthology.orchestra.run.smithi089.stdout: Qpathinfo 157932 1.317 2200.525 2024-09-21T04:21:21.683 INFO:teuthology.orchestra.run.smithi089.stdout: Qfileinfo 27556 0.006 0.475 2024-09-21T04:21:21.684 INFO:teuthology.orchestra.run.smithi089.stdout: Qfsinfo 28852 0.570 14.478 2024-09-21T04:21:21.684 INFO:teuthology.orchestra.run.smithi089.stdout: Sfileinfo 14198 14.396 2193.536 2024-09-21T04:21:21.684 INFO:teuthology.orchestra.run.smithi089.stdout: Find 60936 0.918 21.988 2024-09-21T04:21:21.684 INFO:teuthology.orchestra.run.smithi089.stdout: WriteX 85960 15.329 2383.544 2024-09-21T04:21:21.684 INFO:teuthology.orchestra.run.smithi089.stdout: ReadX 272958 0.010 11.084 2024-09-21T04:21:21.684 INFO:teuthology.orchestra.run.smithi089.stdout: LockX 568 0.341 3.402 2024-09-21T04:21:21.684 INFO:teuthology.orchestra.run.smithi089.stdout: UnlockX 568 0.296 1.509 2024-09-21T04:21:21.684 INFO:teuthology.orchestra.run.smithi089.stdout: Flush 12153 0.003 3.384 2024-09-21T04:21:21.684 INFO:teuthology.orchestra.run.smithi089.stdout: 2024-09-21T04:21:21.684 INFO:teuthology.orchestra.run.smithi089.stdout:Throughput 9.04896 MB/sec 5 clients 5 procs max_latency=2383.554 ms 2024-09-21T04:21:21.686 DEBUG:teuthology.orchestra.run.smithi089:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'umount /mnt/foo' 2024-09-21T04:21:21.810 DEBUG:teuthology.orchestra.run.smithi089:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'while ! mount -t nfs $(hostname):/fake /mnt/foo ; do sleep 5 ; done' 2024-09-21T04:21:21.903 DEBUG:teuthology.orchestra.run.smithi089:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'cd /mnt/foo && dbench 5 -t 5' 2024-09-21T04:21:21.931 INFO:teuthology.orchestra.run.smithi089.stdout:dbench version 4.00 - Copyright Andrew Tridgell 1999-2004 2024-09-21T04:21:21.931 INFO:teuthology.orchestra.run.smithi089.stdout: 2024-09-21T04:21:21.931 INFO:teuthology.orchestra.run.smithi089.stdout:Running for 5 seconds with load '/usr/share/dbench/client.txt' and minimum warmup 1 secs 2024-09-21T04:21:21.932 INFO:teuthology.orchestra.run.smithi089.stdout:4 of 5 processes prepared for launch 0 sec 2024-09-21T04:21:22.015 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:21:21 smithi089 ceph-mon[64076]: osdmap e159: 8 total, 8 up, 8 in 2024-09-21T04:21:22.032 INFO:teuthology.orchestra.run.smithi089.stdout:5 of 5 processes prepared for launch 0 sec 2024-09-21T04:21:22.032 INFO:teuthology.orchestra.run.smithi089.stdout:releasing clients 2024-09-21T04:21:22.281 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:21:21 smithi195 ceph-mon[49435]: osdmap e159: 8 total, 8 up, 8 in 2024-09-21T04:21:22.515 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:21:22 smithi089 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-mon-smithi089[64072]: 2024-09-21T04:21:22.227+0000 7fc2f4fd0640 -1 log_channel(cluster) log [ERR] : Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-09-21T04:21:22.515 INFO:journalctl@ceph.osd.1.smithi089.stdout:Sep 21 04:21:22 smithi089 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-1[71474]: 2024-09-21T04:21:22.273+0000 7f4c86d22640 -1 bdev(0x5588fc95ec00 /var/lib/ceph/osd/ceph-1/block) aio_submit retries 1 2024-09-21T04:21:22.609 INFO:journalctl@ceph.osd.5.smithi195.stdout:Sep 21 04:21:22 smithi195 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-5[55460]: 2024-09-21T04:21:22.277+0000 7f7971a75640 -1 bdev(0x55ccd43c6c00 /var/lib/ceph/osd/ceph-5/block) aio_submit retries 1 2024-09-21T04:21:22.609 INFO:journalctl@ceph.osd.5.smithi195.stdout:Sep 21 04:21:22 smithi195 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-5[55460]: 2024-09-21T04:21:22.278+0000 7f7974a7b640 -1 bdev(0x55ccd43c6c00 /var/lib/ceph/osd/ceph-5/block) aio_submit retries 3 2024-09-21T04:21:23.032 INFO:teuthology.orchestra.run.smithi089.stdout: 0 8 0.00 MB/sec warmup 1 sec latency 999.935 ms 2024-09-21T04:21:23.265 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:21:22 smithi089 ceph-mon[64076]: osdmap e160: 8 total, 8 up, 8 in 2024-09-21T04:21:23.265 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:21:22 smithi089 ceph-mon[64076]: pgmap v309: 97 pgs: 2 remapped+peering, 7 active+undersized, 27 active+undersized+degraded, 61 active+clean; 332 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 5.0 MiB/s wr, 687 op/s; 205/1653 objects degraded (12.402%) 2024-09-21T04:21:23.265 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:21:22 smithi089 ceph-mon[64076]: Health check failed: 1 filesystem is degraded (FS_DEGRADED) 2024-09-21T04:21:23.265 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:21:22 smithi089 ceph-mon[64076]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-09-21T04:21:23.265 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:21:22 smithi089 ceph-mon[64076]: osdmap e161: 8 total, 8 up, 8 in 2024-09-21T04:21:23.265 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:21:22 smithi089 ceph-mon[64076]: Standby daemon mds.foofs.smithi195.hijxre assigned to filesystem foofs as rank 0 2024-09-21T04:21:23.265 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:21:22 smithi089 ceph-mon[64076]: Health check failed: insufficient standby MDS daemons available (MDS_INSUFFICIENT_STANDBY) 2024-09-21T04:21:23.265 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:21:22 smithi089 ceph-mon[64076]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-09-21T04:21:23.266 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:21:22 smithi089 ceph-mon[64076]: fsmap foofs:0/1 1 up:standby, 1 failed 2024-09-21T04:21:23.266 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:21:22 smithi089 ceph-mon[64076]: fsmap foofs:1/1 {0=foofs.smithi195.hijxre=up:replay} 2024-09-21T04:21:23.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:21:22 smithi195 ceph-mon[49435]: osdmap e160: 8 total, 8 up, 8 in 2024-09-21T04:21:23.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:21:22 smithi195 ceph-mon[49435]: pgmap v309: 97 pgs: 2 remapped+peering, 7 active+undersized, 27 active+undersized+degraded, 61 active+clean; 332 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 5.0 MiB/s wr, 687 op/s; 205/1653 objects degraded (12.402%) 2024-09-21T04:21:23.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:21:22 smithi195 ceph-mon[49435]: Health check failed: 1 filesystem is degraded (FS_DEGRADED) 2024-09-21T04:21:23.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:21:22 smithi195 ceph-mon[49435]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-09-21T04:21:23.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:21:22 smithi195 ceph-mon[49435]: osdmap e161: 8 total, 8 up, 8 in 2024-09-21T04:21:23.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:21:22 smithi195 ceph-mon[49435]: Standby daemon mds.foofs.smithi195.hijxre assigned to filesystem foofs as rank 0 2024-09-21T04:21:23.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:21:22 smithi195 ceph-mon[49435]: Health check failed: insufficient standby MDS daemons available (MDS_INSUFFICIENT_STANDBY) 2024-09-21T04:21:23.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:21:22 smithi195 ceph-mon[49435]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-09-21T04:21:23.360 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:21:22 smithi195 ceph-mon[49435]: fsmap foofs:0/1 1 up:standby, 1 failed 2024-09-21T04:21:23.360 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:21:22 smithi195 ceph-mon[49435]: fsmap foofs:1/1 {0=foofs.smithi195.hijxre=up:replay} 2024-09-21T04:21:24.032 INFO:teuthology.orchestra.run.smithi089.stdout: 0 8 0.00 MB/sec warmup 2 sec latency 2000.011 ms 2024-09-21T04:21:24.264 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:21:23 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "mds metadata", "who": "foofs.smithi089.osufws"} : dispatch 2024-09-21T04:21:24.265 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:21:23 smithi089 ceph-mon[64076]: osdmap e162: 8 total, 8 up, 8 in 2024-09-21T04:21:24.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:21:23 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "mds metadata", "who": "foofs.smithi089.osufws"} : dispatch 2024-09-21T04:21:24.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:21:23 smithi195 ceph-mon[49435]: osdmap e162: 8 total, 8 up, 8 in 2024-09-21T04:21:25.032 INFO:teuthology.orchestra.run.smithi089.stdout: 0 8 0.00 MB/sec warmup 3 sec latency 3000.061 ms 2024-09-21T04:21:25.264 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:21:24 smithi089 ceph-mon[64076]: reconnect by client.24345 172.21.15.195:0/1361098494 after 0.000999982 2024-09-21T04:21:25.265 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:21:24 smithi089 ceph-mon[64076]: reconnect by client.14564 172.21.15.89:0/227406872 after 0.000999982 2024-09-21T04:21:25.265 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:21:24 smithi089 ceph-mon[64076]: mds.? [v2:172.21.15.195:6832/3071482597,v1:172.21.15.195:6833/3071482597] up:reconnect 2024-09-21T04:21:25.265 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:21:24 smithi089 ceph-mon[64076]: fsmap foofs:1/1 {0=foofs.smithi195.hijxre=up:reconnect} 2024-09-21T04:21:25.265 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:21:24 smithi089 ceph-mon[64076]: pgmap v312: 97 pgs: 1 peering, 3 active+undersized+degraded, 93 active+clean; 326 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 35 MiB/s rd, 2.1 MiB/s wr, 259 op/s; 30/1527 objects degraded (1.965%); 4.0 MiB/s, 70 objects/s recovering 2024-09-21T04:21:25.265 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:21:24 smithi089 ceph-mon[64076]: Health check update: Degraded data redundancy: 205/1653 objects degraded (12.402%), 27 pgs degraded (PG_DEGRADED) 2024-09-21T04:21:25.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:21:24 smithi195 ceph-mon[49435]: reconnect by client.24345 172.21.15.195:0/1361098494 after 0.000999982 2024-09-21T04:21:25.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:21:24 smithi195 ceph-mon[49435]: reconnect by client.14564 172.21.15.89:0/227406872 after 0.000999982 2024-09-21T04:21:25.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:21:24 smithi195 ceph-mon[49435]: mds.? [v2:172.21.15.195:6832/3071482597,v1:172.21.15.195:6833/3071482597] up:reconnect 2024-09-21T04:21:25.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:21:24 smithi195 ceph-mon[49435]: fsmap foofs:1/1 {0=foofs.smithi195.hijxre=up:reconnect} 2024-09-21T04:21:25.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:21:24 smithi195 ceph-mon[49435]: pgmap v312: 97 pgs: 1 peering, 3 active+undersized+degraded, 93 active+clean; 326 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 35 MiB/s rd, 2.1 MiB/s wr, 259 op/s; 30/1527 objects degraded (1.965%); 4.0 MiB/s, 70 objects/s recovering 2024-09-21T04:21:25.360 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:21:24 smithi195 ceph-mon[49435]: Health check update: Degraded data redundancy: 205/1653 objects degraded (12.402%), 27 pgs degraded (PG_DEGRADED) 2024-09-21T04:21:26.032 INFO:teuthology.orchestra.run.smithi089.stdout: 0 8 0.00 MB/sec warmup 4 sec latency 4000.094 ms 2024-09-21T04:21:26.265 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:21:25 smithi089 ceph-mon[64076]: mds.? [v2:172.21.15.195:6832/3071482597,v1:172.21.15.195:6833/3071482597] up:rejoin 2024-09-21T04:21:26.265 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:21:25 smithi089 ceph-mon[64076]: fsmap foofs:1/1 {0=foofs.smithi195.hijxre=up:rejoin} 2024-09-21T04:21:26.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:21:25 smithi195 ceph-mon[49435]: mds.? [v2:172.21.15.195:6832/3071482597,v1:172.21.15.195:6833/3071482597] up:rejoin 2024-09-21T04:21:26.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:21:25 smithi195 ceph-mon[49435]: fsmap foofs:1/1 {0=foofs.smithi195.hijxre=up:rejoin} 2024-09-21T04:21:27.032 INFO:teuthology.orchestra.run.smithi089.stdout:true 2024-09-21T04:21:27.033 INFO:teuthology.orchestra.run.smithi089.stdout: 0 8 0.00 MB/sec warmup 5 sec latency 5000.205 ms 2024-09-21T04:21:27.215 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:21:26 smithi089 ceph-mon[64076]: mds.? [v2:172.21.15.195:6832/3071482597,v1:172.21.15.195:6833/3071482597] up:clientreplay 2024-09-21T04:21:27.215 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:21:26 smithi089 ceph-mon[64076]: fsmap foofs:1/1 {0=foofs.smithi195.hijxre=up:clientreplay} 2024-09-21T04:21:27.215 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:21:26 smithi089 ceph-mon[64076]: daemon mds.foofs.smithi195.hijxre is now active in filesystem foofs as rank 0 2024-09-21T04:21:27.215 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:21:26 smithi089 ceph-mon[64076]: pgmap v313: 97 pgs: 1 peering, 96 active+clean; 326 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 36 MiB/s rd, 1.1 MiB/s wr, 102 op/s; 3.2 MiB/s, 66 objects/s recovering 2024-09-21T04:21:27.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:21:26 smithi195 ceph-mon[49435]: mds.? [v2:172.21.15.195:6832/3071482597,v1:172.21.15.195:6833/3071482597] up:clientreplay 2024-09-21T04:21:27.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:21:26 smithi195 ceph-mon[49435]: fsmap foofs:1/1 {0=foofs.smithi195.hijxre=up:clientreplay} 2024-09-21T04:21:27.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:21:26 smithi195 ceph-mon[49435]: daemon mds.foofs.smithi195.hijxre is now active in filesystem foofs as rank 0 2024-09-21T04:21:27.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:21:26 smithi195 ceph-mon[49435]: pgmap v313: 97 pgs: 1 peering, 96 active+clean; 326 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 36 MiB/s rd, 1.1 MiB/s wr, 102 op/s; 3.2 MiB/s, 66 objects/s recovering 2024-09-21T04:21:27.950 INFO:teuthology.orchestra.run.smithi089.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-21T04:21:27.950 INFO:teuthology.orchestra.run.smithi089.stdout:alertmanager.smithi089 smithi089 *:9093,9094 running (9m) 2m ago 18m 17.9M - 0.27.0 11f11916f8cd 65dfcbcafee1 2024-09-21T04:21:27.950 INFO:teuthology.orchestra.run.smithi089.stdout:crash.smithi089 smithi089 running (6m) 2m ago 18m 6891k - 19.3.0-5103-gf36283e7 48ed0a2cf89d 6c415a5e173e 2024-09-21T04:21:27.950 INFO:teuthology.orchestra.run.smithi089.stdout:crash.smithi195 smithi195 running (6m) 16s ago 18m 6891k - 19.3.0-5103-gf36283e7 48ed0a2cf89d 78ce05a4e9ab 2024-09-21T04:21:27.950 INFO:teuthology.orchestra.run.smithi089.stdout:grafana.smithi089 smithi089 *:3000 running (9m) 2m ago 18m 47.2M - dad864ee21e9 ec4d6738bce7 2024-09-21T04:21:27.951 INFO:teuthology.orchestra.run.smithi089.stdout:mds.foofs.smithi089.osufws smithi089 running (12m) 2m ago 12m 38.5M - 17.2.0 e1d6a67b021e 664d41859bea 2024-09-21T04:21:27.951 INFO:teuthology.orchestra.run.smithi089.stdout:mds.foofs.smithi195.hijxre smithi195 running (12m) 16s ago 12m 17.8M - 17.2.0 e1d6a67b021e 7bc0c6542d12 2024-09-21T04:21:27.951 INFO:teuthology.orchestra.run.smithi089.stdout:mgr.smithi089.srlyxf smithi089 *:8443,9283,8765 running (8m) 2m ago 19m 519M - 19.3.0-5103-gf36283e7 48ed0a2cf89d 3cbeaaa88241 2024-09-21T04:21:27.951 INFO:teuthology.orchestra.run.smithi089.stdout:mgr.smithi195.cgotzm smithi195 *:8443,9283,8765 running (7m) 16s ago 18m 458M - 19.3.0-5103-gf36283e7 48ed0a2cf89d d3734574cf66 2024-09-21T04:21:27.951 INFO:teuthology.orchestra.run.smithi089.stdout:mon.smithi089 smithi089 running (7m) 2m ago 19m 53.6M 2048M 19.3.0-5103-gf36283e7 48ed0a2cf89d 7dd34c8a6f21 2024-09-21T04:21:27.951 INFO:teuthology.orchestra.run.smithi089.stdout:mon.smithi195 smithi195 running (7m) 16s ago 17m 47.1M 2048M 19.3.0-5103-gf36283e7 48ed0a2cf89d 4255937444e7 2024-09-21T04:21:27.951 INFO:teuthology.orchestra.run.smithi089.stdout:nfs.foo.0.0.smithi089.dcjmqc smithi089 *:2049 running (12m) 2m ago 12m 290M - 4.0 e1d6a67b021e 98550f94e773 2024-09-21T04:21:27.951 INFO:teuthology.orchestra.run.smithi089.stdout:nfs.foo.1.0.smithi195.vhoayp smithi195 *:2049 running (12m) 16s ago 12m 90.7M - 4.0 e1d6a67b021e c60a4040130a 2024-09-21T04:21:27.951 INFO:teuthology.orchestra.run.smithi089.stdout:node-exporter.smithi089 smithi089 *:9100 running (9m) 2m ago 18m 9.84M - 1.7.0 72c9c2088986 7ef0d1588475 2024-09-21T04:21:27.951 INFO:teuthology.orchestra.run.smithi089.stdout:node-exporter.smithi195 smithi195 *:9100 running (8m) 16s ago 17m 10.3M - 1.7.0 72c9c2088986 61eafd61bc6c 2024-09-21T04:21:27.951 INFO:teuthology.orchestra.run.smithi089.stdout:osd.0 smithi089 running (5m) 2m ago 17m 331M 1935M 19.3.0-5103-gf36283e7 48ed0a2cf89d 3df0adea59fb 2024-09-21T04:21:27.951 INFO:teuthology.orchestra.run.smithi089.stdout:osd.1 smithi089 running (4m) 2m ago 16m 265M 1935M 19.3.0-5103-gf36283e7 48ed0a2cf89d da2e49774e7e 2024-09-21T04:21:27.951 INFO:teuthology.orchestra.run.smithi089.stdout:osd.2 smithi089 running (3m) 2m ago 16m 202M 1935M 19.3.0-5103-gf36283e7 48ed0a2cf89d b9f6438dd5b9 2024-09-21T04:21:27.952 INFO:teuthology.orchestra.run.smithi089.stdout:osd.3 smithi089 running (3m) 2m ago 15m 11.5M 1935M 19.3.0-5103-gf36283e7 48ed0a2cf89d d269cd94bd08 2024-09-21T04:21:27.952 INFO:teuthology.orchestra.run.smithi089.stdout:osd.4 smithi195 running (2m) 16s ago 15m 436M 2704M 19.3.0-5103-gf36283e7 48ed0a2cf89d e375f90f95c4 2024-09-21T04:21:27.952 INFO:teuthology.orchestra.run.smithi089.stdout:osd.5 smithi195 running (100s) 16s ago 15m 316M 2704M 19.3.0-5103-gf36283e7 48ed0a2cf89d 41de0beb713d 2024-09-21T04:21:27.952 INFO:teuthology.orchestra.run.smithi089.stdout:osd.6 smithi195 running (60s) 16s ago 14m 267M 2704M 19.3.0-5103-gf36283e7 48ed0a2cf89d a41d0c0574ab 2024-09-21T04:21:27.952 INFO:teuthology.orchestra.run.smithi089.stdout:osd.7 smithi195 running (20s) 16s ago 14m 12.1M 2704M 19.3.0-5103-gf36283e7 48ed0a2cf89d 66d3bf5f6c5f 2024-09-21T04:21:27.952 INFO:teuthology.orchestra.run.smithi089.stdout:prometheus.smithi089 smithi089 *:9095 running (7m) 2m ago 17m 53.3M - 2.51.0 1d3b7f56885b 8c9f309a6459 2024-09-21T04:21:28.028 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:21:27 smithi089 ceph-mon[64076]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 30/1527 objects degraded (1.965%), 3 pgs degraded) 2024-09-21T04:21:28.028 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:21:27 smithi089 ceph-mon[64076]: Health check cleared: FS_DEGRADED (was: 1 filesystem is degraded) 2024-09-21T04:21:28.028 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:21:27 smithi089 ceph-mon[64076]: mds.? [v2:172.21.15.195:6832/3071482597,v1:172.21.15.195:6833/3071482597] up:active 2024-09-21T04:21:28.028 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:21:27 smithi089 ceph-mon[64076]: fsmap foofs:1 {0=foofs.smithi195.hijxre=up:active} 2024-09-21T04:21:28.028 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:21:27 smithi089 ceph-mon[64076]: from='client.34472 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T04:21:28.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:21:27 smithi195 ceph-mon[49435]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 30/1527 objects degraded (1.965%), 3 pgs degraded) 2024-09-21T04:21:28.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:21:27 smithi195 ceph-mon[49435]: Health check cleared: FS_DEGRADED (was: 1 filesystem is degraded) 2024-09-21T04:21:28.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:21:27 smithi195 ceph-mon[49435]: mds.? [v2:172.21.15.195:6832/3071482597,v1:172.21.15.195:6833/3071482597] up:active 2024-09-21T04:21:28.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:21:27 smithi195 ceph-mon[49435]: fsmap foofs:1 {0=foofs.smithi195.hijxre=up:active} 2024-09-21T04:21:28.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:21:27 smithi195 ceph-mon[49435]: from='client.34472 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T04:21:28.488 INFO:teuthology.orchestra.run.smithi089.stdout:{ 2024-09-21T04:21:28.488 INFO:teuthology.orchestra.run.smithi089.stdout: "mon": { 2024-09-21T04:21:28.488 INFO:teuthology.orchestra.run.smithi089.stdout: "ceph version 19.3.0-5103-gf36283e7 (f36283e7c91cac1c01cb9a7c4ff21bd697dbd4f2) squid (dev)": 2 2024-09-21T04:21:28.488 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2024-09-21T04:21:28.488 INFO:teuthology.orchestra.run.smithi089.stdout: "mgr": { 2024-09-21T04:21:28.488 INFO:teuthology.orchestra.run.smithi089.stdout: "ceph version 19.3.0-5103-gf36283e7 (f36283e7c91cac1c01cb9a7c4ff21bd697dbd4f2) squid (dev)": 2 2024-09-21T04:21:28.488 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2024-09-21T04:21:28.488 INFO:teuthology.orchestra.run.smithi089.stdout: "osd": { 2024-09-21T04:21:28.488 INFO:teuthology.orchestra.run.smithi089.stdout: "ceph version 19.3.0-5103-gf36283e7 (f36283e7c91cac1c01cb9a7c4ff21bd697dbd4f2) squid (dev)": 8 2024-09-21T04:21:28.488 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2024-09-21T04:21:28.488 INFO:teuthology.orchestra.run.smithi089.stdout: "mds": { 2024-09-21T04:21:28.488 INFO:teuthology.orchestra.run.smithi089.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 1 2024-09-21T04:21:28.488 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2024-09-21T04:21:28.488 INFO:teuthology.orchestra.run.smithi089.stdout: "overall": { 2024-09-21T04:21:28.489 INFO:teuthology.orchestra.run.smithi089.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 1, 2024-09-21T04:21:28.489 INFO:teuthology.orchestra.run.smithi089.stdout: "ceph version 19.3.0-5103-gf36283e7 (f36283e7c91cac1c01cb9a7c4ff21bd697dbd4f2) squid (dev)": 12 2024-09-21T04:21:28.489 INFO:teuthology.orchestra.run.smithi089.stdout: } 2024-09-21T04:21:28.489 INFO:teuthology.orchestra.run.smithi089.stdout:} 2024-09-21T04:21:28.996 INFO:teuthology.orchestra.run.smithi089.stdout:{ 2024-09-21T04:21:28.996 INFO:teuthology.orchestra.run.smithi089.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:2195e5f178be0a9390728e2a14088236d8fe6f82635f5c84418fe37a04457e00", 2024-09-21T04:21:28.996 INFO:teuthology.orchestra.run.smithi089.stdout: "in_progress": true, 2024-09-21T04:21:28.996 INFO:teuthology.orchestra.run.smithi089.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-21T04:21:28.997 INFO:teuthology.orchestra.run.smithi089.stdout: "services_complete": [ 2024-09-21T04:21:28.997 INFO:teuthology.orchestra.run.smithi089.stdout: "mgr", 2024-09-21T04:21:28.997 INFO:teuthology.orchestra.run.smithi089.stdout: "osd", 2024-09-21T04:21:28.997 INFO:teuthology.orchestra.run.smithi089.stdout: "crash", 2024-09-21T04:21:28.997 INFO:teuthology.orchestra.run.smithi089.stdout: "mon" 2024-09-21T04:21:28.997 INFO:teuthology.orchestra.run.smithi089.stdout: ], 2024-09-21T04:21:28.997 INFO:teuthology.orchestra.run.smithi089.stdout: "progress": "14/23 daemons upgraded", 2024-09-21T04:21:28.997 INFO:teuthology.orchestra.run.smithi089.stdout: "message": "Currently upgrading mds daemons", 2024-09-21T04:21:28.997 INFO:teuthology.orchestra.run.smithi089.stdout: "is_paused": false 2024-09-21T04:21:28.997 INFO:teuthology.orchestra.run.smithi089.stdout:} 2024-09-21T04:21:29.032 INFO:teuthology.orchestra.run.smithi089.stdout: 5 269 15.35 MB/sec execute 1 sec latency 136.454 ms 2024-09-21T04:21:29.094 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:21:28 smithi089 ceph-mon[64076]: from='client.34476 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T04:21:29.094 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:21:28 smithi089 ceph-mon[64076]: from='client.34480 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T04:21:29.094 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:21:28 smithi089 ceph-mon[64076]: pgmap v314: 97 pgs: 97 active+clean; 299 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 36 MiB/s rd, 1.1 MiB/s wr, 104 op/s; 2.7 MiB/s, 56 objects/s recovering 2024-09-21T04:21:29.094 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:21:28 smithi089 ceph-mon[64076]: from='client.? 172.21.15.89:0/3713019623' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T04:21:29.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:21:28 smithi195 ceph-mon[49435]: from='client.34476 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T04:21:29.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:21:28 smithi195 ceph-mon[49435]: from='client.34480 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T04:21:29.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:21:28 smithi195 ceph-mon[49435]: pgmap v314: 97 pgs: 97 active+clean; 299 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 36 MiB/s rd, 1.1 MiB/s wr, 104 op/s; 2.7 MiB/s, 56 objects/s recovering 2024-09-21T04:21:29.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:21:28 smithi195 ceph-mon[49435]: from='client.? 172.21.15.89:0/3713019623' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T04:21:29.509 INFO:teuthology.orchestra.run.smithi089.stdout:HEALTH_WARN insufficient standby MDS daemons available 2024-09-21T04:21:29.509 INFO:teuthology.orchestra.run.smithi089.stdout:[WRN] MDS_INSUFFICIENT_STANDBY: insufficient standby MDS daemons available 2024-09-21T04:21:29.509 INFO:teuthology.orchestra.run.smithi089.stdout: have 0; want 1 more 2024-09-21T04:21:30.032 INFO:teuthology.orchestra.run.smithi089.stdout: 5 344 13.39 MB/sec execute 2 sec latency 118.169 ms 2024-09-21T04:21:30.184 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:21:29 smithi089 ceph-mon[64076]: from='client.34488 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T04:21:30.184 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:21:29 smithi089 ceph-mon[64076]: from='client.? 172.21.15.89:0/1880618101' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-21T04:21:30.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:21:29 smithi195 ceph-mon[49435]: from='client.34488 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T04:21:30.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:21:29 smithi195 ceph-mon[49435]: from='client.? 172.21.15.89:0/1880618101' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-21T04:21:31.032 INFO:teuthology.orchestra.run.smithi089.stdout: 5 423 13.18 MB/sec execute 3 sec latency 127.823 ms 2024-09-21T04:21:31.211 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:21:30 smithi089 ceph-mon[64076]: pgmap v315: 97 pgs: 97 active+clean; 255 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 27 MiB/s rd, 4.5 MiB/s wr, 140 op/s; 2.0 MiB/s, 42 objects/s recovering 2024-09-21T04:21:31.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:21:30 smithi195 ceph-mon[49435]: pgmap v315: 97 pgs: 97 active+clean; 255 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 27 MiB/s rd, 4.5 MiB/s wr, 140 op/s; 2.0 MiB/s, 42 objects/s recovering 2024-09-21T04:21:32.033 INFO:teuthology.orchestra.run.smithi089.stdout: 5 493 12.68 MB/sec execute 4 sec latency 124.256 ms 2024-09-21T04:21:33.033 INFO:teuthology.orchestra.run.smithi089.stdout: 5 cleanup 5 sec 2024-09-21T04:21:33.327 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:21:33 smithi089 ceph-mon[64076]: pgmap v316: 97 pgs: 97 active+clean; 257 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 22 MiB/s rd, 4.3 MiB/s wr, 189 op/s; 1.7 MiB/s, 34 objects/s recovering 2024-09-21T04:21:33.327 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:21:33 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:21:33.327 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:21:33 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T04:21:33.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:21:33 smithi195 ceph-mon[49435]: pgmap v316: 97 pgs: 97 active+clean; 257 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 22 MiB/s rd, 4.3 MiB/s wr, 189 op/s; 1.7 MiB/s, 34 objects/s recovering 2024-09-21T04:21:33.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:21:33 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:21:33.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:21:33 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T04:21:33.661 INFO:teuthology.orchestra.run.smithi089.stdout: 0 cleanup 6 sec 2024-09-21T04:21:33.661 INFO:teuthology.orchestra.run.smithi089.stdout: 2024-09-21T04:21:33.661 INFO:teuthology.orchestra.run.smithi089.stdout: Operation Count AvgLat MaxLat 2024-09-21T04:21:33.661 INFO:teuthology.orchestra.run.smithi089.stdout: ---------------------------------------- 2024-09-21T04:21:33.661 INFO:teuthology.orchestra.run.smithi089.stdout: NTCreateX 314 41.997 127.819 2024-09-21T04:21:33.661 INFO:teuthology.orchestra.run.smithi089.stdout: Close 300 18.327 90.281 2024-09-21T04:21:33.661 INFO:teuthology.orchestra.run.smithi089.stdout: Rename 1 62.006 62.006 2024-09-21T04:21:33.661 INFO:teuthology.orchestra.run.smithi089.stdout: Unlink 6 8.196 16.895 2024-09-21T04:21:33.661 INFO:teuthology.orchestra.run.smithi089.stdout: Qpathinfo 27 2.540 29.704 2024-09-21T04:21:33.661 INFO:teuthology.orchestra.run.smithi089.stdout: Qfileinfo 295 18.134 78.486 2024-09-21T04:21:33.661 INFO:teuthology.orchestra.run.smithi089.stdout: Qfsinfo 39 0.486 1.071 2024-09-21T04:21:33.661 INFO:teuthology.orchestra.run.smithi089.stdout: Sfileinfo 18 18.375 48.683 2024-09-21T04:21:33.662 INFO:teuthology.orchestra.run.smithi089.stdout: Find 36 0.988 2.881 2024-09-21T04:21:33.662 INFO:teuthology.orchestra.run.smithi089.stdout: WriteX 1407 0.029 0.075 2024-09-21T04:21:33.662 INFO:teuthology.orchestra.run.smithi089.stdout: ReadX 130 0.022 1.038 2024-09-21T04:21:33.662 INFO:teuthology.orchestra.run.smithi089.stdout: Flush 29 19.177 37.339 2024-09-21T04:21:33.662 INFO:teuthology.orchestra.run.smithi089.stdout: 2024-09-21T04:21:33.662 INFO:teuthology.orchestra.run.smithi089.stdout:Throughput 12.6794 MB/sec 5 clients 5 procs max_latency=136.454 ms 2024-09-21T04:21:33.663 DEBUG:teuthology.parallel:result is None 2024-09-21T04:21:35.515 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:21:35 smithi089 ceph-mon[64076]: pgmap v317: 97 pgs: 97 active+clean; 308 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 14 MiB/s rd, 8.4 MiB/s wr, 194 op/s; 1.5 MiB/s, 24 objects/s recovering 2024-09-21T04:21:35.515 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:21:35 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:21:35.515 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:21:35 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:21:35.515 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:21:35 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T04:21:35.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:21:35 smithi195 ceph-mon[49435]: pgmap v317: 97 pgs: 97 active+clean; 308 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 14 MiB/s rd, 8.4 MiB/s wr, 194 op/s; 1.5 MiB/s, 24 objects/s recovering 2024-09-21T04:21:35.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:21:35 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:21:35.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:21:35 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:21:35.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:21:35 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T04:21:36.406 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:21:36 smithi089 ceph-mon[64076]: Health check cleared: MDS_INSUFFICIENT_STANDBY (was: insufficient standby MDS daemons available) 2024-09-21T04:21:36.406 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:21:36 smithi089 ceph-mon[64076]: Cluster is now healthy 2024-09-21T04:21:36.406 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:21:36 smithi089 ceph-mon[64076]: mds.? [v2:172.21.15.89:6834/1700388577,v1:172.21.15.89:6835/1700388577] up:boot 2024-09-21T04:21:36.406 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:21:36 smithi089 ceph-mon[64076]: fsmap foofs:1 {0=foofs.smithi195.hijxre=up:active} 1 up:standby 2024-09-21T04:21:36.406 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:21:36 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "mds metadata", "who": "foofs.smithi089.osufws"} : dispatch 2024-09-21T04:21:36.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:21:36 smithi195 ceph-mon[49435]: Health check cleared: MDS_INSUFFICIENT_STANDBY (was: insufficient standby MDS daemons available) 2024-09-21T04:21:36.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:21:36 smithi195 ceph-mon[49435]: Cluster is now healthy 2024-09-21T04:21:36.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:21:36 smithi195 ceph-mon[49435]: mds.? [v2:172.21.15.89:6834/1700388577,v1:172.21.15.89:6835/1700388577] up:boot 2024-09-21T04:21:36.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:21:36 smithi195 ceph-mon[49435]: fsmap foofs:1 {0=foofs.smithi195.hijxre=up:active} 1 up:standby 2024-09-21T04:21:36.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:21:36 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "mds metadata", "who": "foofs.smithi089.osufws"} : dispatch 2024-09-21T04:21:37.510 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:21:37 smithi195 ceph-mon[49435]: pgmap v318: 97 pgs: 97 active+clean; 316 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 6.4 MiB/s rd, 8.6 MiB/s wr, 205 op/s; 6.9 KiB/s, 5 objects/s recovering 2024-09-21T04:21:37.514 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:21:37 smithi089 ceph-mon[64076]: pgmap v318: 97 pgs: 97 active+clean; 316 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 6.4 MiB/s rd, 8.6 MiB/s wr, 205 op/s; 6.9 KiB/s, 5 objects/s recovering 2024-09-21T04:21:38.750 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:21:38 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:21:38.750 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:21:38 smithi195 ceph-mon[49435]: pgmap v319: 97 pgs: 97 active+clean; 295 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 2.7 MiB/s rd, 9.0 MiB/s wr, 214 op/s 2024-09-21T04:21:38.765 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:21:38 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:21:38.765 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:21:38 smithi089 ceph-mon[64076]: pgmap v319: 97 pgs: 97 active+clean; 295 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 2.7 MiB/s rd, 9.0 MiB/s wr, 214 op/s 2024-09-21T04:21:41.109 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:21:40 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:21:41.109 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:21:40 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:21:41.109 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:21:40 smithi195 ceph-mon[49435]: pgmap v320: 97 pgs: 97 active+clean; 218 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 32 KiB/s rd, 8.9 MiB/s wr, 222 op/s 2024-09-21T04:21:41.110 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:21:40 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:21:41.110 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:21:40 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:21:41.164 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:21:40 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:21:41.164 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:21:40 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:21:41.164 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:21:40 smithi089 ceph-mon[64076]: pgmap v320: 97 pgs: 97 active+clean; 218 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 32 KiB/s rd, 8.9 MiB/s wr, 222 op/s 2024-09-21T04:21:41.165 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:21:40 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:21:41.165 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:21:40 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:21:43.319 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:21:43 smithi089 ceph-mon[64076]: pgmap v321: 97 pgs: 97 active+clean; 218 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 20 KiB/s rd, 6.5 MiB/s wr, 182 op/s 2024-09-21T04:21:43.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:21:43 smithi195 ceph-mon[49435]: pgmap v321: 97 pgs: 97 active+clean; 218 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 20 KiB/s rd, 6.5 MiB/s wr, 182 op/s 2024-09-21T04:21:45.265 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:21:44 smithi089 ceph-mon[64076]: Detected new or changed devices on smithi089 2024-09-21T04:21:45.265 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:21:44 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:21:45.265 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:21:44 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:21:45.265 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:21:44 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T04:21:45.265 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:21:44 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T04:21:45.265 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:21:44 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:21:45.265 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:21:44 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:21:45.265 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:21:44 smithi089 ceph-mon[64076]: pgmap v322: 97 pgs: 97 active+clean; 218 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 16 KiB/s rd, 5.9 MiB/s wr, 122 op/s 2024-09-21T04:21:45.265 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:21:44 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T04:21:45.265 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:21:44 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "versions"} : dispatch 2024-09-21T04:21:45.265 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:21:44 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "versions"} : dispatch 2024-09-21T04:21:45.265 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:21:44 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "versions"} : dispatch 2024-09-21T04:21:45.266 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:21:44 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "versions"} : dispatch 2024-09-21T04:21:45.266 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:21:44 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "mds ok-to-stop", "ids": ["foofs.smithi195.hijxre"]} : dispatch 2024-09-21T04:21:45.360 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:21:44 smithi195 ceph-mon[49435]: Detected new or changed devices on smithi089 2024-09-21T04:21:45.360 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:21:44 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:21:45.360 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:21:44 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:21:45.360 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:21:44 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T04:21:45.360 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:21:44 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T04:21:45.360 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:21:44 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:21:45.360 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:21:44 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:21:45.360 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:21:44 smithi195 ceph-mon[49435]: pgmap v322: 97 pgs: 97 active+clean; 218 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 16 KiB/s rd, 5.9 MiB/s wr, 122 op/s 2024-09-21T04:21:45.360 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:21:44 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T04:21:45.361 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:21:44 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "versions"} : dispatch 2024-09-21T04:21:45.361 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:21:44 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "versions"} : dispatch 2024-09-21T04:21:45.361 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:21:44 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "versions"} : dispatch 2024-09-21T04:21:45.361 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:21:44 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "versions"} : dispatch 2024-09-21T04:21:45.361 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:21:44 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "mds ok-to-stop", "ids": ["foofs.smithi195.hijxre"]} : dispatch 2024-09-21T04:21:46.197 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:21:46 smithi195 ceph-mon[49435]: Upgrade: It appears safe to stop mds.foofs.smithi195.hijxre 2024-09-21T04:21:46.198 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:21:46 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:21:46.198 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:21:46 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "auth get-or-create", "entity": "mds.foofs.smithi195.hijxre", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-09-21T04:21:46.198 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:21:46 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T04:21:46.265 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:21:45 smithi089 ceph-mon[64076]: Upgrade: It appears safe to stop mds.foofs.smithi195.hijxre 2024-09-21T04:21:46.265 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:21:45 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:21:46.265 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:21:45 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "auth get-or-create", "entity": "mds.foofs.smithi195.hijxre", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-09-21T04:21:46.265 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:21:45 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T04:21:47.264 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:21:47 smithi089 ceph-mon[64076]: Upgrade: Updating mds.foofs.smithi195.hijxre 2024-09-21T04:21:47.265 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:21:47 smithi089 ceph-mon[64076]: Deploying daemon mds.foofs.smithi195.hijxre on smithi195 2024-09-21T04:21:47.265 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:21:47 smithi089 ceph-mon[64076]: pgmap v323: 97 pgs: 97 active+clean; 218 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 4.0 KiB/s rd, 1.7 MiB/s wr, 83 op/s 2024-09-21T04:21:47.290 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:21:47 smithi195 ceph-mon[49435]: Upgrade: Updating mds.foofs.smithi195.hijxre 2024-09-21T04:21:47.290 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:21:47 smithi195 ceph-mon[49435]: Deploying daemon mds.foofs.smithi195.hijxre on smithi195 2024-09-21T04:21:47.290 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:21:47 smithi195 ceph-mon[49435]: pgmap v323: 97 pgs: 97 active+clean; 218 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 4.0 KiB/s rd, 1.7 MiB/s wr, 83 op/s 2024-09-21T04:21:47.609 INFO:journalctl@ceph.osd.6.smithi195.stdout:Sep 21 04:21:47 smithi195 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-6[57554]: 2024-09-21T04:21:47.368+0000 7f9856264640 -1 bdev(0x5599339cec00 /var/lib/ceph/osd/ceph-6/block) aio_submit retries 1 2024-09-21T04:21:47.609 INFO:journalctl@ceph.osd.6.smithi195.stdout:Sep 21 04:21:47 smithi195 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-6[57554]: 2024-09-21T04:21:47.368+0000 7f985a26c640 -1 bdev(0x5599339cec00 /var/lib/ceph/osd/ceph-6/block) aio_submit retries 1 2024-09-21T04:21:47.765 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:21:47 smithi089 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-mon-smithi089[64072]: 2024-09-21T04:21:47.290+0000 7fc2f4fd0640 -1 log_channel(cluster) log [ERR] : Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-09-21T04:21:48.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:21:48 smithi195 ceph-mon[49435]: Health check failed: 1 filesystem is degraded (FS_DEGRADED) 2024-09-21T04:21:48.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:21:48 smithi195 ceph-mon[49435]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-09-21T04:21:48.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:21:48 smithi195 ceph-mon[49435]: osdmap e163: 8 total, 8 up, 8 in 2024-09-21T04:21:48.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:21:48 smithi195 ceph-mon[49435]: Standby daemon mds.foofs.smithi089.osufws assigned to filesystem foofs as rank 0 2024-09-21T04:21:48.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:21:48 smithi195 ceph-mon[49435]: Health check failed: insufficient standby MDS daemons available (MDS_INSUFFICIENT_STANDBY) 2024-09-21T04:21:48.360 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:21:48 smithi195 ceph-mon[49435]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-09-21T04:21:48.360 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:21:48 smithi195 ceph-mon[49435]: fsmap foofs:0/1 1 up:standby, 1 failed 2024-09-21T04:21:48.360 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:21:48 smithi195 ceph-mon[49435]: fsmap foofs:1/1 {0=foofs.smithi089.osufws=up:replay} 2024-09-21T04:21:48.360 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:21:48 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:21:48.360 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:21:48 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T04:21:48.360 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:21:48 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "mds metadata", "who": "foofs.smithi195.hijxre"} : dispatch 2024-09-21T04:21:48.515 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:21:48 smithi089 ceph-mon[64076]: Health check failed: 1 filesystem is degraded (FS_DEGRADED) 2024-09-21T04:21:48.515 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:21:48 smithi089 ceph-mon[64076]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-09-21T04:21:48.515 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:21:48 smithi089 ceph-mon[64076]: osdmap e163: 8 total, 8 up, 8 in 2024-09-21T04:21:48.515 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:21:48 smithi089 ceph-mon[64076]: Standby daemon mds.foofs.smithi089.osufws assigned to filesystem foofs as rank 0 2024-09-21T04:21:48.515 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:21:48 smithi089 ceph-mon[64076]: Health check failed: insufficient standby MDS daemons available (MDS_INSUFFICIENT_STANDBY) 2024-09-21T04:21:48.515 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:21:48 smithi089 ceph-mon[64076]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-09-21T04:21:48.515 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:21:48 smithi089 ceph-mon[64076]: fsmap foofs:0/1 1 up:standby, 1 failed 2024-09-21T04:21:48.515 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:21:48 smithi089 ceph-mon[64076]: fsmap foofs:1/1 {0=foofs.smithi089.osufws=up:replay} 2024-09-21T04:21:48.515 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:21:48 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:21:48.516 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:21:48 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T04:21:48.516 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:21:48 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "mds metadata", "who": "foofs.smithi195.hijxre"} : dispatch 2024-09-21T04:21:49.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:21:49 smithi195 ceph-mon[49435]: pgmap v325: 97 pgs: 97 active+clean; 218 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 820 KiB/s rd, 20 KiB/s wr, 23 op/s 2024-09-21T04:21:49.515 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:21:49 smithi089 ceph-mon[64076]: pgmap v325: 97 pgs: 97 active+clean; 218 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 820 KiB/s rd, 20 KiB/s wr, 23 op/s 2024-09-21T04:21:50.109 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:21:50 smithi195 ceph-mon[49435]: reconnect by client.14564 172.21.15.89:0/227406872 after 0 2024-09-21T04:21:50.109 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:21:50 smithi195 ceph-mon[49435]: reconnect by client.24345 172.21.15.195:0/1361098494 after 0.000999979 2024-09-21T04:21:50.109 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:21:50 smithi195 ceph-mon[49435]: mds.? [v2:172.21.15.89:6834/1700388577,v1:172.21.15.89:6835/1700388577] up:reconnect 2024-09-21T04:21:50.109 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:21:50 smithi195 ceph-mon[49435]: fsmap foofs:1/1 {0=foofs.smithi089.osufws=up:reconnect} 2024-09-21T04:21:50.515 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:21:50 smithi089 ceph-mon[64076]: reconnect by client.14564 172.21.15.89:0/227406872 after 0 2024-09-21T04:21:50.515 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:21:50 smithi089 ceph-mon[64076]: reconnect by client.24345 172.21.15.195:0/1361098494 after 0.000999979 2024-09-21T04:21:50.515 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:21:50 smithi089 ceph-mon[64076]: mds.? [v2:172.21.15.89:6834/1700388577,v1:172.21.15.89:6835/1700388577] up:reconnect 2024-09-21T04:21:50.515 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:21:50 smithi089 ceph-mon[64076]: fsmap foofs:1/1 {0=foofs.smithi089.osufws=up:reconnect} 2024-09-21T04:21:51.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:21:51 smithi195 ceph-mon[49435]: mds.? [v2:172.21.15.89:6834/1700388577,v1:172.21.15.89:6835/1700388577] up:rejoin 2024-09-21T04:21:51.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:21:51 smithi195 ceph-mon[49435]: fsmap foofs:1/1 {0=foofs.smithi089.osufws=up:rejoin} 2024-09-21T04:21:51.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:21:51 smithi195 ceph-mon[49435]: daemon mds.foofs.smithi089.osufws is now active in filesystem foofs as rank 0 2024-09-21T04:21:51.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:21:51 smithi195 ceph-mon[49435]: pgmap v326: 97 pgs: 97 active+clean; 218 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 13 MiB/s rd, 20 KiB/s wr, 6 op/s 2024-09-21T04:21:51.376 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:21:51 smithi089 ceph-mon[64076]: mds.? [v2:172.21.15.89:6834/1700388577,v1:172.21.15.89:6835/1700388577] up:rejoin 2024-09-21T04:21:51.376 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:21:51 smithi089 ceph-mon[64076]: fsmap foofs:1/1 {0=foofs.smithi089.osufws=up:rejoin} 2024-09-21T04:21:51.377 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:21:51 smithi089 ceph-mon[64076]: daemon mds.foofs.smithi089.osufws is now active in filesystem foofs as rank 0 2024-09-21T04:21:51.377 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:21:51 smithi089 ceph-mon[64076]: pgmap v326: 97 pgs: 97 active+clean; 218 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 13 MiB/s rd, 20 KiB/s wr, 6 op/s 2024-09-21T04:21:52.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:21:52 smithi195 ceph-mon[49435]: Health check cleared: FS_DEGRADED (was: 1 filesystem is degraded) 2024-09-21T04:21:52.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:21:52 smithi195 ceph-mon[49435]: mds.? [v2:172.21.15.89:6834/1700388577,v1:172.21.15.89:6835/1700388577] up:active 2024-09-21T04:21:52.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:21:52 smithi195 ceph-mon[49435]: fsmap foofs:1 {0=foofs.smithi089.osufws=up:active} 2024-09-21T04:21:52.515 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:21:52 smithi089 ceph-mon[64076]: Health check cleared: FS_DEGRADED (was: 1 filesystem is degraded) 2024-09-21T04:21:52.515 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:21:52 smithi089 ceph-mon[64076]: mds.? [v2:172.21.15.89:6834/1700388577,v1:172.21.15.89:6835/1700388577] up:active 2024-09-21T04:21:52.515 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:21:52 smithi089 ceph-mon[64076]: fsmap foofs:1 {0=foofs.smithi089.osufws=up:active} 2024-09-21T04:21:53.330 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:21:53 smithi195 ceph-mon[49435]: pgmap v327: 97 pgs: 97 active+clean; 218 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 17 MiB/s rd, 20 KiB/s wr, 7 op/s 2024-09-21T04:21:53.514 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:21:53 smithi089 ceph-mon[64076]: pgmap v327: 97 pgs: 97 active+clean; 218 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 17 MiB/s rd, 20 KiB/s wr, 7 op/s 2024-09-21T04:21:55.100 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:21:54 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:21:55.100 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:21:54 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:21:55.100 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:21:54 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T04:21:55.100 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:21:54 smithi195 ceph-mon[49435]: Health check cleared: MDS_INSUFFICIENT_STANDBY (was: insufficient standby MDS daemons available) 2024-09-21T04:21:55.100 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:21:54 smithi195 ceph-mon[49435]: Cluster is now healthy 2024-09-21T04:21:55.101 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:21:54 smithi195 ceph-mon[49435]: mds.? [v2:172.21.15.195:6832/2945281874,v1:172.21.15.195:6833/2945281874] up:boot 2024-09-21T04:21:55.101 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:21:54 smithi195 ceph-mon[49435]: fsmap foofs:1 {0=foofs.smithi089.osufws=up:active} 1 up:standby 2024-09-21T04:21:55.101 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:21:54 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "mds metadata", "who": "foofs.smithi195.hijxre"} : dispatch 2024-09-21T04:21:55.101 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:21:54 smithi195 ceph-mon[49435]: pgmap v328: 97 pgs: 97 active+clean; 218 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 21 MiB/s rd, 409 B/s wr, 9 op/s 2024-09-21T04:21:55.110 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:21:54 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:21:55.111 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:21:54 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:21:55.111 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:21:54 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T04:21:55.111 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:21:54 smithi089 ceph-mon[64076]: Health check cleared: MDS_INSUFFICIENT_STANDBY (was: insufficient standby MDS daemons available) 2024-09-21T04:21:55.111 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:21:54 smithi089 ceph-mon[64076]: Cluster is now healthy 2024-09-21T04:21:55.111 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:21:54 smithi089 ceph-mon[64076]: mds.? [v2:172.21.15.195:6832/2945281874,v1:172.21.15.195:6833/2945281874] up:boot 2024-09-21T04:21:55.111 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:21:54 smithi089 ceph-mon[64076]: fsmap foofs:1 {0=foofs.smithi089.osufws=up:active} 1 up:standby 2024-09-21T04:21:55.111 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:21:54 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "mds metadata", "who": "foofs.smithi195.hijxre"} : dispatch 2024-09-21T04:21:55.111 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:21:54 smithi089 ceph-mon[64076]: pgmap v328: 97 pgs: 97 active+clean; 218 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 21 MiB/s rd, 409 B/s wr, 9 op/s 2024-09-21T04:21:57.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:21:57 smithi195 ceph-mon[49435]: pgmap v329: 97 pgs: 97 active+clean; 218 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 21 MiB/s rd, 409 B/s wr, 11 op/s 2024-09-21T04:21:57.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:21:57 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:21:57.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:21:57 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:21:57.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:21:57 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:21:57.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:21:57 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:21:57.515 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:21:57 smithi089 ceph-mon[64076]: pgmap v329: 97 pgs: 97 active+clean; 218 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 21 MiB/s rd, 409 B/s wr, 11 op/s 2024-09-21T04:21:57.515 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:21:57 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:21:57.515 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:21:57 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:21:57.515 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:21:57 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:21:57.515 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:21:57 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:21:59.515 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:21:59 smithi089 ceph-mon[64076]: pgmap v330: 97 pgs: 97 active+clean; 218 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 20 MiB/s rd, 8.3 KiB/s wr, 14 op/s 2024-09-21T04:21:59.577 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:21:59 smithi195 ceph-mon[49435]: pgmap v330: 97 pgs: 97 active+clean; 218 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 20 MiB/s rd, 8.3 KiB/s wr, 14 op/s 2024-09-21T04:21:59.883 INFO:teuthology.orchestra.run.smithi089.stdout:true 2024-09-21T04:22:00.596 INFO:teuthology.orchestra.run.smithi089.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-21T04:22:00.596 INFO:teuthology.orchestra.run.smithi089.stdout:alertmanager.smithi089 smithi089 *:9093,9094 running (10m) 20s ago 19m 22.0M - 0.27.0 11f11916f8cd 65dfcbcafee1 2024-09-21T04:22:00.596 INFO:teuthology.orchestra.run.smithi089.stdout:crash.smithi089 smithi089 running (6m) 20s ago 19m 6891k - 19.3.0-5103-gf36283e7 48ed0a2cf89d 6c415a5e173e 2024-09-21T04:22:00.596 INFO:teuthology.orchestra.run.smithi089.stdout:crash.smithi195 smithi195 running (6m) 4s ago 18m 6891k - 19.3.0-5103-gf36283e7 48ed0a2cf89d 78ce05a4e9ab 2024-09-21T04:22:00.596 INFO:teuthology.orchestra.run.smithi089.stdout:grafana.smithi089 smithi089 *:3000 running (10m) 20s ago 18m 55.7M - dad864ee21e9 ec4d6738bce7 2024-09-21T04:22:00.596 INFO:teuthology.orchestra.run.smithi089.stdout:mds.foofs.smithi089.osufws smithi089 running (26s) 20s ago 13m 15.3M - 19.3.0-5103-gf36283e7 48ed0a2cf89d 526fde771b8f 2024-09-21T04:22:00.596 INFO:teuthology.orchestra.run.smithi089.stdout:mds.foofs.smithi195.hijxre smithi195 running (7s) 4s ago 13m 11.7M - 19.3.0-5103-gf36283e7 48ed0a2cf89d 9b532cb9643e 2024-09-21T04:22:00.596 INFO:teuthology.orchestra.run.smithi089.stdout:mgr.smithi089.srlyxf smithi089 *:8443,9283,8765 running (9m) 20s ago 20m 523M - 19.3.0-5103-gf36283e7 48ed0a2cf89d 3cbeaaa88241 2024-09-21T04:22:00.596 INFO:teuthology.orchestra.run.smithi089.stdout:mgr.smithi195.cgotzm smithi195 *:8443,9283,8765 running (8m) 4s ago 18m 458M - 19.3.0-5103-gf36283e7 48ed0a2cf89d d3734574cf66 2024-09-21T04:22:00.596 INFO:teuthology.orchestra.run.smithi089.stdout:mon.smithi089 smithi089 running (8m) 20s ago 20m 63.6M 2048M 19.3.0-5103-gf36283e7 48ed0a2cf89d 7dd34c8a6f21 2024-09-21T04:22:00.596 INFO:teuthology.orchestra.run.smithi089.stdout:mon.smithi195 smithi195 running (7m) 4s ago 18m 49.7M 2048M 19.3.0-5103-gf36283e7 48ed0a2cf89d 4255937444e7 2024-09-21T04:22:00.597 INFO:teuthology.orchestra.run.smithi089.stdout:nfs.foo.0.0.smithi089.dcjmqc smithi089 *:2049 running (12m) 20s ago 12m 273M - 4.0 e1d6a67b021e 98550f94e773 2024-09-21T04:22:00.597 INFO:teuthology.orchestra.run.smithi089.stdout:nfs.foo.1.0.smithi195.vhoayp smithi195 *:2049 running (12m) 4s ago 12m 90.8M - 4.0 e1d6a67b021e c60a4040130a 2024-09-21T04:22:00.597 INFO:teuthology.orchestra.run.smithi089.stdout:node-exporter.smithi089 smithi089 *:9100 running (9m) 20s ago 18m 11.1M - 1.7.0 72c9c2088986 7ef0d1588475 2024-09-21T04:22:00.597 INFO:teuthology.orchestra.run.smithi089.stdout:node-exporter.smithi195 smithi195 *:9100 running (9m) 4s ago 18m 11.3M - 1.7.0 72c9c2088986 61eafd61bc6c 2024-09-21T04:22:00.597 INFO:teuthology.orchestra.run.smithi089.stdout:osd.0 smithi089 running (6m) 20s ago 17m 432M 1935M 19.3.0-5103-gf36283e7 48ed0a2cf89d 3df0adea59fb 2024-09-21T04:22:00.597 INFO:teuthology.orchestra.run.smithi089.stdout:osd.1 smithi089 running (5m) 20s ago 17m 328M 1935M 19.3.0-5103-gf36283e7 48ed0a2cf89d da2e49774e7e 2024-09-21T04:22:00.597 INFO:teuthology.orchestra.run.smithi089.stdout:osd.2 smithi089 running (4m) 20s ago 16m 320M 1935M 19.3.0-5103-gf36283e7 48ed0a2cf89d b9f6438dd5b9 2024-09-21T04:22:00.597 INFO:teuthology.orchestra.run.smithi089.stdout:osd.3 smithi089 running (3m) 20s ago 16m 372M 1935M 19.3.0-5103-gf36283e7 48ed0a2cf89d d269cd94bd08 2024-09-21T04:22:00.597 INFO:teuthology.orchestra.run.smithi089.stdout:osd.4 smithi195 running (2m) 4s ago 15m 467M 2704M 19.3.0-5103-gf36283e7 48ed0a2cf89d e375f90f95c4 2024-09-21T04:22:00.597 INFO:teuthology.orchestra.run.smithi089.stdout:osd.5 smithi195 running (2m) 4s ago 15m 339M 2704M 19.3.0-5103-gf36283e7 48ed0a2cf89d 41de0beb713d 2024-09-21T04:22:00.597 INFO:teuthology.orchestra.run.smithi089.stdout:osd.6 smithi195 running (93s) 4s ago 15m 284M 2704M 19.3.0-5103-gf36283e7 48ed0a2cf89d a41d0c0574ab 2024-09-21T04:22:00.597 INFO:teuthology.orchestra.run.smithi089.stdout:osd.7 smithi195 running (53s) 4s ago 14m 251M 2704M 19.3.0-5103-gf36283e7 48ed0a2cf89d 66d3bf5f6c5f 2024-09-21T04:22:00.597 INFO:teuthology.orchestra.run.smithi089.stdout:prometheus.smithi089 smithi089 *:9095 running (8m) 20s ago 18m 57.8M - 2.51.0 1d3b7f56885b 8c9f309a6459 2024-09-21T04:22:01.065 INFO:teuthology.orchestra.run.smithi089.stdout:{ 2024-09-21T04:22:01.066 INFO:teuthology.orchestra.run.smithi089.stdout: "mon": { 2024-09-21T04:22:01.066 INFO:teuthology.orchestra.run.smithi089.stdout: "ceph version 19.3.0-5103-gf36283e7 (f36283e7c91cac1c01cb9a7c4ff21bd697dbd4f2) squid (dev)": 2 2024-09-21T04:22:01.066 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2024-09-21T04:22:01.066 INFO:teuthology.orchestra.run.smithi089.stdout: "mgr": { 2024-09-21T04:22:01.066 INFO:teuthology.orchestra.run.smithi089.stdout: "ceph version 19.3.0-5103-gf36283e7 (f36283e7c91cac1c01cb9a7c4ff21bd697dbd4f2) squid (dev)": 2 2024-09-21T04:22:01.066 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2024-09-21T04:22:01.066 INFO:teuthology.orchestra.run.smithi089.stdout: "osd": { 2024-09-21T04:22:01.067 INFO:teuthology.orchestra.run.smithi089.stdout: "ceph version 19.3.0-5103-gf36283e7 (f36283e7c91cac1c01cb9a7c4ff21bd697dbd4f2) squid (dev)": 8 2024-09-21T04:22:01.067 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2024-09-21T04:22:01.067 INFO:teuthology.orchestra.run.smithi089.stdout: "mds": { 2024-09-21T04:22:01.067 INFO:teuthology.orchestra.run.smithi089.stdout: "ceph version 19.3.0-5103-gf36283e7 (f36283e7c91cac1c01cb9a7c4ff21bd697dbd4f2) squid (dev)": 2 2024-09-21T04:22:01.067 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2024-09-21T04:22:01.067 INFO:teuthology.orchestra.run.smithi089.stdout: "overall": { 2024-09-21T04:22:01.067 INFO:teuthology.orchestra.run.smithi089.stdout: "ceph version 19.3.0-5103-gf36283e7 (f36283e7c91cac1c01cb9a7c4ff21bd697dbd4f2) squid (dev)": 14 2024-09-21T04:22:01.068 INFO:teuthology.orchestra.run.smithi089.stdout: } 2024-09-21T04:22:01.068 INFO:teuthology.orchestra.run.smithi089.stdout:} 2024-09-21T04:22:01.266 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:22:01 smithi089 ceph-mon[64076]: from='client.34502 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T04:22:01.266 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:22:01 smithi089 ceph-mon[64076]: pgmap v331: 97 pgs: 97 active+clean; 218 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 17 MiB/s rd, 7.2 KiB/s wr, 11 op/s 2024-09-21T04:22:01.266 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:22:01 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:22:01.266 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:22:01 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:22:01.266 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:22:01 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T04:22:01.266 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:22:01 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T04:22:01.266 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:22:01 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:22:01.266 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:22:01 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:22:01.266 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:22:01 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T04:22:01.266 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:22:01 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "versions"} : dispatch 2024-09-21T04:22:01.267 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:22:01 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "versions"} : dispatch 2024-09-21T04:22:01.267 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:22:01 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "versions"} : dispatch 2024-09-21T04:22:01.267 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:22:01 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "versions"} : dispatch 2024-09-21T04:22:01.267 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:22:01 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "versions"} : dispatch 2024-09-21T04:22:01.267 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:22:01 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:22:01.267 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:22:01 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi089.osufws"} : dispatch 2024-09-21T04:22:01.267 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:22:01 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi089.osufws"}]': finished 2024-09-21T04:22:01.267 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:22:01 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi195.hijxre"} : dispatch 2024-09-21T04:22:01.267 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:22:01 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi195.hijxre"}]': finished 2024-09-21T04:22:01.267 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:22:01 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "versions"} : dispatch 2024-09-21T04:22:01.267 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:22:01 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:22:01.267 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:22:01 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "versions"} : dispatch 2024-09-21T04:22:01.268 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:22:01 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:22:01.268 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:22:01 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "versions"} : dispatch 2024-09-21T04:22:01.268 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:22:01 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "versions"} : dispatch 2024-09-21T04:22:01.268 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:22:01 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:22:01.268 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:22:01 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "versions"} : dispatch 2024-09-21T04:22:01.268 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:22:01 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:22:01.268 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:22:01 smithi089 ceph-mon[64076]: from='client.? 172.21.15.89:0/1999807178' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T04:22:01.434 INFO:teuthology.orchestra.run.smithi089.stdout:{ 2024-09-21T04:22:01.434 INFO:teuthology.orchestra.run.smithi089.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:2195e5f178be0a9390728e2a14088236d8fe6f82635f5c84418fe37a04457e00", 2024-09-21T04:22:01.434 INFO:teuthology.orchestra.run.smithi089.stdout: "in_progress": true, 2024-09-21T04:22:01.434 INFO:teuthology.orchestra.run.smithi089.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-21T04:22:01.434 INFO:teuthology.orchestra.run.smithi089.stdout: "services_complete": [ 2024-09-21T04:22:01.434 INFO:teuthology.orchestra.run.smithi089.stdout: "osd", 2024-09-21T04:22:01.434 INFO:teuthology.orchestra.run.smithi089.stdout: "crash", 2024-09-21T04:22:01.435 INFO:teuthology.orchestra.run.smithi089.stdout: "mgr", 2024-09-21T04:22:01.435 INFO:teuthology.orchestra.run.smithi089.stdout: "mds", 2024-09-21T04:22:01.435 INFO:teuthology.orchestra.run.smithi089.stdout: "mon" 2024-09-21T04:22:01.435 INFO:teuthology.orchestra.run.smithi089.stdout: ], 2024-09-21T04:22:01.435 INFO:teuthology.orchestra.run.smithi089.stdout: "progress": "16/23 daemons upgraded", 2024-09-21T04:22:01.435 INFO:teuthology.orchestra.run.smithi089.stdout: "message": "Currently upgrading nfs daemons", 2024-09-21T04:22:01.435 INFO:teuthology.orchestra.run.smithi089.stdout: "is_paused": false 2024-09-21T04:22:01.435 INFO:teuthology.orchestra.run.smithi089.stdout:} 2024-09-21T04:22:01.551 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:22:01 smithi195 ceph-mon[49435]: from='client.34502 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T04:22:01.551 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:22:01 smithi195 ceph-mon[49435]: pgmap v331: 97 pgs: 97 active+clean; 218 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 17 MiB/s rd, 7.2 KiB/s wr, 11 op/s 2024-09-21T04:22:01.551 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:22:01 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:22:01.551 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:22:01 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:22:01.551 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:22:01 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T04:22:01.551 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:22:01 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T04:22:01.551 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:22:01 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:22:01.551 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:22:01 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:22:01.551 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:22:01 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T04:22:01.552 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:22:01 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "versions"} : dispatch 2024-09-21T04:22:01.552 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:22:01 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "versions"} : dispatch 2024-09-21T04:22:01.552 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:22:01 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "versions"} : dispatch 2024-09-21T04:22:01.552 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:22:01 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "versions"} : dispatch 2024-09-21T04:22:01.552 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:22:01 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "versions"} : dispatch 2024-09-21T04:22:01.552 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:22:01 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:22:01.552 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:22:01 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi089.osufws"} : dispatch 2024-09-21T04:22:01.552 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:22:01 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi089.osufws"}]': finished 2024-09-21T04:22:01.552 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:22:01 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi195.hijxre"} : dispatch 2024-09-21T04:22:01.552 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:22:01 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi195.hijxre"}]': finished 2024-09-21T04:22:01.552 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:22:01 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "versions"} : dispatch 2024-09-21T04:22:01.552 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:22:01 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:22:01.552 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:22:01 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "versions"} : dispatch 2024-09-21T04:22:01.552 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:22:01 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:22:01.552 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:22:01 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "versions"} : dispatch 2024-09-21T04:22:01.553 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:22:01 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "versions"} : dispatch 2024-09-21T04:22:01.553 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:22:01 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:22:01.553 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:22:01 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "versions"} : dispatch 2024-09-21T04:22:01.553 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:22:01 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:22:01.553 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:22:01 smithi195 ceph-mon[49435]: from='client.? 172.21.15.89:0/1999807178' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T04:22:01.874 INFO:teuthology.orchestra.run.smithi089.stdout:HEALTH_OK 2024-09-21T04:22:02.520 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:22:02 smithi089 ceph-mon[64076]: from='client.34506 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T04:22:02.520 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:22:02 smithi089 ceph-mon[64076]: Detected new or changed devices on smithi195 2024-09-21T04:22:02.520 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:22:02 smithi089 ceph-mon[64076]: Upgrade: Setting container_image for all mds 2024-09-21T04:22:02.520 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:22:02 smithi089 ceph-mon[64076]: from='client.34510 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T04:22:02.520 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:22:02 smithi089 ceph-mon[64076]: Upgrade: Setting container_image for all rgw 2024-09-21T04:22:02.520 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:22:02 smithi089 ceph-mon[64076]: Upgrade: Setting container_image for all rbd-mirror 2024-09-21T04:22:02.521 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:22:02 smithi089 ceph-mon[64076]: Upgrade: Setting container_image for all ceph-exporter 2024-09-21T04:22:02.521 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:22:02 smithi089 ceph-mon[64076]: Upgrade: Setting container_image for all iscsi 2024-09-21T04:22:02.521 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:22:02 smithi089 ceph-mon[64076]: from='client.? 172.21.15.89:0/2701093619' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-21T04:22:02.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:22:02 smithi195 ceph-mon[49435]: from='client.34506 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T04:22:02.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:22:02 smithi195 ceph-mon[49435]: Detected new or changed devices on smithi195 2024-09-21T04:22:02.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:22:02 smithi195 ceph-mon[49435]: Upgrade: Setting container_image for all mds 2024-09-21T04:22:02.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:22:02 smithi195 ceph-mon[49435]: from='client.34510 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T04:22:02.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:22:02 smithi195 ceph-mon[49435]: Upgrade: Setting container_image for all rgw 2024-09-21T04:22:02.610 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:22:02 smithi195 ceph-mon[49435]: Upgrade: Setting container_image for all rbd-mirror 2024-09-21T04:22:02.610 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:22:02 smithi195 ceph-mon[49435]: Upgrade: Setting container_image for all ceph-exporter 2024-09-21T04:22:02.610 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:22:02 smithi195 ceph-mon[49435]: Upgrade: Setting container_image for all iscsi 2024-09-21T04:22:02.610 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:22:02 smithi195 ceph-mon[49435]: from='client.? 172.21.15.89:0/2701093619' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-21T04:22:03.480 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:22:03 smithi089 ceph-mon[64076]: from='client.34518 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T04:22:03.480 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:22:03 smithi089 ceph-mon[64076]: pgmap v332: 97 pgs: 97 active+clean; 218 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 6.7 MiB/s rd, 7.2 KiB/s wr, 8 op/s 2024-09-21T04:22:03.480 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:22:03 smithi089 ceph-mon[64076]: Upgrade: Updating nfs.foo.0.0.smithi089.dcjmqc (1/2) 2024-09-21T04:22:03.480 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:22:03 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:22:03.480 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:22:03 smithi089 ceph-mon[64076]: Creating key for client.nfs.foo.0.0.smithi089.dcjmqc 2024-09-21T04:22:03.480 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:22:03 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi089.dcjmqc", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]} : dispatch 2024-09-21T04:22:03.480 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:22:03 smithi089 ceph-mon[64076]: Ensuring nfs.foo.0 is in the ganesha grace table 2024-09-21T04:22:03.480 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:22:03 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]} : dispatch 2024-09-21T04:22:03.480 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:22:03 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-09-21T04:22:03.481 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:22:03 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T04:22:03.481 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:22:03 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"} : dispatch 2024-09-21T04:22:03.481 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:22:03 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-09-21T04:22:03.481 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:22:03 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi089.dcjmqc-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]} : dispatch 2024-09-21T04:22:03.481 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:22:03 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T04:22:03.481 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:22:03 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:22:03.481 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:22:03 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T04:22:03.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:22:03 smithi195 ceph-mon[49435]: from='client.34518 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T04:22:03.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:22:03 smithi195 ceph-mon[49435]: pgmap v332: 97 pgs: 97 active+clean; 218 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 6.7 MiB/s rd, 7.2 KiB/s wr, 8 op/s 2024-09-21T04:22:03.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:22:03 smithi195 ceph-mon[49435]: Upgrade: Updating nfs.foo.0.0.smithi089.dcjmqc (1/2) 2024-09-21T04:22:03.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:22:03 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:22:03.610 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:22:03 smithi195 ceph-mon[49435]: Creating key for client.nfs.foo.0.0.smithi089.dcjmqc 2024-09-21T04:22:03.610 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:22:03 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi089.dcjmqc", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]} : dispatch 2024-09-21T04:22:03.610 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:22:03 smithi195 ceph-mon[49435]: Ensuring nfs.foo.0 is in the ganesha grace table 2024-09-21T04:22:03.610 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:22:03 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]} : dispatch 2024-09-21T04:22:03.610 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:22:03 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-09-21T04:22:03.610 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:22:03 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T04:22:03.610 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:22:03 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"} : dispatch 2024-09-21T04:22:03.610 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:22:03 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-09-21T04:22:03.610 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:22:03 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi089.dcjmqc-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]} : dispatch 2024-09-21T04:22:03.610 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:22:03 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T04:22:03.610 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:22:03 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:22:03.610 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:22:03 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T04:22:04.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:22:04 smithi195 ceph-mon[49435]: Rados config object exists: conf-nfs.foo 2024-09-21T04:22:04.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:22:04 smithi195 ceph-mon[49435]: Creating key for client.nfs.foo.0.0.smithi089.dcjmqc-rgw 2024-09-21T04:22:04.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:22:04 smithi195 ceph-mon[49435]: Bind address in nfs.foo.0.0.smithi089.dcjmqc's ganesha conf is defaulting to empty 2024-09-21T04:22:04.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:22:04 smithi195 ceph-mon[49435]: Deploying daemon nfs.foo.0.0.smithi089.dcjmqc on smithi089 2024-09-21T04:22:04.511 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:22:04 smithi089 ceph-mon[64076]: Rados config object exists: conf-nfs.foo 2024-09-21T04:22:04.511 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:22:04 smithi089 ceph-mon[64076]: Creating key for client.nfs.foo.0.0.smithi089.dcjmqc-rgw 2024-09-21T04:22:04.511 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:22:04 smithi089 ceph-mon[64076]: Bind address in nfs.foo.0.0.smithi089.dcjmqc's ganesha conf is defaulting to empty 2024-09-21T04:22:04.511 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:22:04 smithi089 ceph-mon[64076]: Deploying daemon nfs.foo.0.0.smithi089.dcjmqc on smithi089 2024-09-21T04:22:05.354 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:22:05 smithi089 ceph-mon[64076]: pgmap v333: 97 pgs: 97 active+clean; 218 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 3.7 MiB/s rd, 7.2 KiB/s wr, 7 op/s 2024-09-21T04:22:05.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:22:05 smithi195 ceph-mon[49435]: pgmap v333: 97 pgs: 97 active+clean; 218 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 3.7 MiB/s rd, 7.2 KiB/s wr, 7 op/s 2024-09-21T04:22:06.469 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:22:06 smithi195 ceph-mon[49435]: Evicting (and blocklisting) client session 14564 (172.21.15.89:0/227406872) 2024-09-21T04:22:06.469 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:22:06 smithi195 ceph-mon[49435]: from='mds.? [v2:172.21.15.89:6834/1700388577,v1:172.21.15.89:6835/1700388577]' entity='mds.foofs.smithi089.osufws' cmd={"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.89:0/227406872"} : dispatch 2024-09-21T04:22:06.470 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:22:06 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:22:06.470 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:22:06 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:22:06.515 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:22:06 smithi089 ceph-mon[64076]: Evicting (and blocklisting) client session 14564 (172.21.15.89:0/227406872) 2024-09-21T04:22:06.515 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:22:06 smithi089 ceph-mon[64076]: from='mds.? [v2:172.21.15.89:6834/1700388577,v1:172.21.15.89:6835/1700388577]' entity='mds.foofs.smithi089.osufws' cmd={"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.89:0/227406872"} : dispatch 2024-09-21T04:22:06.515 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:22:06 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:22:06.515 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:22:06 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:22:07.323 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:22:07 smithi195 ceph-mon[49435]: pgmap v334: 97 pgs: 97 active+clean; 218 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 1.8 KiB/s rd, 7.2 KiB/s wr, 4 op/s 2024-09-21T04:22:07.324 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:22:07 smithi195 ceph-mon[49435]: from='mds.? [v2:172.21.15.89:6834/1700388577,v1:172.21.15.89:6835/1700388577]' entity='mds.foofs.smithi089.osufws' cmd='[{"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.89:0/227406872"}]': finished 2024-09-21T04:22:07.324 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:22:07 smithi195 ceph-mon[49435]: osdmap e164: 8 total, 8 up, 8 in 2024-09-21T04:22:07.515 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:22:07 smithi089 ceph-mon[64076]: pgmap v334: 97 pgs: 97 active+clean; 218 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 1.8 KiB/s rd, 7.2 KiB/s wr, 4 op/s 2024-09-21T04:22:07.515 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:22:07 smithi089 ceph-mon[64076]: from='mds.? [v2:172.21.15.89:6834/1700388577,v1:172.21.15.89:6835/1700388577]' entity='mds.foofs.smithi089.osufws' cmd='[{"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.89:0/227406872"}]': finished 2024-09-21T04:22:07.515 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:22:07 smithi089 ceph-mon[64076]: osdmap e164: 8 total, 8 up, 8 in 2024-09-21T04:22:08.515 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:22:08 smithi089 ceph-mon[64076]: Upgrade: Updating nfs.foo.1.0.smithi195.vhoayp (2/2) 2024-09-21T04:22:08.515 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:22:08 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:22:08.515 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:22:08 smithi089 ceph-mon[64076]: Creating key for client.nfs.foo.1.0.smithi195.vhoayp 2024-09-21T04:22:08.515 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:22:08 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi195.vhoayp", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]} : dispatch 2024-09-21T04:22:08.515 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:22:08 smithi089 ceph-mon[64076]: Ensuring nfs.foo.1 is in the ganesha grace table 2024-09-21T04:22:08.515 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:22:08 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]} : dispatch 2024-09-21T04:22:08.515 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:22:08 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-09-21T04:22:08.515 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:22:08 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T04:22:08.516 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:22:08 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"} : dispatch 2024-09-21T04:22:08.516 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:22:08 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-09-21T04:22:08.516 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:22:08 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi195.vhoayp-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]} : dispatch 2024-09-21T04:22:08.516 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:22:08 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T04:22:08.570 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:22:08 smithi195 ceph-mon[49435]: Upgrade: Updating nfs.foo.1.0.smithi195.vhoayp (2/2) 2024-09-21T04:22:08.570 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:22:08 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:22:08.570 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:22:08 smithi195 ceph-mon[49435]: Creating key for client.nfs.foo.1.0.smithi195.vhoayp 2024-09-21T04:22:08.570 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:22:08 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi195.vhoayp", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]} : dispatch 2024-09-21T04:22:08.570 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:22:08 smithi195 ceph-mon[49435]: Ensuring nfs.foo.1 is in the ganesha grace table 2024-09-21T04:22:08.570 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:22:08 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]} : dispatch 2024-09-21T04:22:08.570 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:22:08 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-09-21T04:22:08.570 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:22:08 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T04:22:08.570 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:22:08 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"} : dispatch 2024-09-21T04:22:08.571 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:22:08 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-09-21T04:22:08.571 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:22:08 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi195.vhoayp-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]} : dispatch 2024-09-21T04:22:08.571 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:22:08 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T04:22:09.456 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:22:09 smithi195 ceph-mon[49435]: Rados config object exists: conf-nfs.foo 2024-09-21T04:22:09.456 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:22:09 smithi195 ceph-mon[49435]: Creating key for client.nfs.foo.1.0.smithi195.vhoayp-rgw 2024-09-21T04:22:09.456 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:22:09 smithi195 ceph-mon[49435]: Bind address in nfs.foo.1.0.smithi195.vhoayp's ganesha conf is defaulting to empty 2024-09-21T04:22:09.456 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:22:09 smithi195 ceph-mon[49435]: Deploying daemon nfs.foo.1.0.smithi195.vhoayp on smithi195 2024-09-21T04:22:09.456 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:22:09 smithi195 ceph-mon[49435]: pgmap v336: 97 pgs: 97 active+clean; 218 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 2.6 KiB/s rd, 1.3 KiB/s wr, 3 op/s 2024-09-21T04:22:09.515 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:22:09 smithi089 ceph-mon[64076]: Rados config object exists: conf-nfs.foo 2024-09-21T04:22:09.515 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:22:09 smithi089 ceph-mon[64076]: Creating key for client.nfs.foo.1.0.smithi195.vhoayp-rgw 2024-09-21T04:22:09.515 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:22:09 smithi089 ceph-mon[64076]: Bind address in nfs.foo.1.0.smithi195.vhoayp's ganesha conf is defaulting to empty 2024-09-21T04:22:09.515 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:22:09 smithi089 ceph-mon[64076]: Deploying daemon nfs.foo.1.0.smithi195.vhoayp on smithi195 2024-09-21T04:22:09.515 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:22:09 smithi089 ceph-mon[64076]: pgmap v336: 97 pgs: 97 active+clean; 218 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 2.6 KiB/s rd, 1.3 KiB/s wr, 3 op/s 2024-09-21T04:22:11.515 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:22:11 smithi089 ceph-mon[64076]: pgmap v337: 97 pgs: 97 active+clean; 218 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 2.6 KiB/s rd, 1.4 KiB/s wr, 4 op/s 2024-09-21T04:22:11.515 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:22:11 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:22:11.515 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:22:11 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:22:11.515 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:22:11 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T04:22:11.541 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:22:11 smithi195 ceph-mon[49435]: pgmap v337: 97 pgs: 97 active+clean; 218 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 2.6 KiB/s rd, 1.4 KiB/s wr, 4 op/s 2024-09-21T04:22:11.542 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:22:11 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:22:11.542 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:22:11 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:22:11.542 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:22:11 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T04:22:13.514 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:22:13 smithi089 ceph-mon[64076]: pgmap v338: 97 pgs: 97 active+clean; 218 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 2.6 KiB/s rd, 1.4 KiB/s wr, 4 op/s 2024-09-21T04:22:13.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:22:13 smithi195 ceph-mon[49435]: pgmap v338: 97 pgs: 97 active+clean; 218 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 2.6 KiB/s rd, 1.4 KiB/s wr, 4 op/s 2024-09-21T04:22:14.808 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:22:14 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:22:14.808 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:22:14 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:22:14.808 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:22:14 smithi195 ceph-mon[49435]: from='mds.? [v2:172.21.15.89:6834/1700388577,v1:172.21.15.89:6835/1700388577]' entity='mds.foofs.smithi089.osufws' cmd={"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.195:0/1361098494"} : dispatch 2024-09-21T04:22:14.808 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:22:14 smithi195 ceph-mon[49435]: pgmap v339: 97 pgs: 97 active+clean; 218 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 5.1 KiB/s rd, 2.2 KiB/s wr, 7 op/s 2024-09-21T04:22:14.809 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:22:14 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:22:14.809 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:22:14 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:22:15.015 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:22:14 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:22:15.015 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:22:14 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:22:15.015 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:22:14 smithi089 ceph-mon[64076]: from='mds.? [v2:172.21.15.89:6834/1700388577,v1:172.21.15.89:6835/1700388577]' entity='mds.foofs.smithi089.osufws' cmd={"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.195:0/1361098494"} : dispatch 2024-09-21T04:22:15.015 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:22:14 smithi089 ceph-mon[64076]: pgmap v339: 97 pgs: 97 active+clean; 218 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 5.1 KiB/s rd, 2.2 KiB/s wr, 7 op/s 2024-09-21T04:22:15.015 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:22:14 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:22:15.015 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:22:14 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:22:15.859 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:22:15 smithi195 ceph-mon[49435]: Evicting (and blocklisting) client session 24345 (172.21.15.195:0/1361098494) 2024-09-21T04:22:15.859 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:22:15 smithi195 ceph-mon[49435]: from='mds.? [v2:172.21.15.89:6834/1700388577,v1:172.21.15.89:6835/1700388577]' entity='mds.foofs.smithi089.osufws' cmd='[{"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.195:0/1361098494"}]': finished 2024-09-21T04:22:15.859 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:22:15 smithi195 ceph-mon[49435]: osdmap e165: 8 total, 8 up, 8 in 2024-09-21T04:22:16.015 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:22:15 smithi089 ceph-mon[64076]: Evicting (and blocklisting) client session 24345 (172.21.15.195:0/1361098494) 2024-09-21T04:22:16.015 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:22:15 smithi089 ceph-mon[64076]: from='mds.? [v2:172.21.15.89:6834/1700388577,v1:172.21.15.89:6835/1700388577]' entity='mds.foofs.smithi089.osufws' cmd='[{"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.195:0/1361098494"}]': finished 2024-09-21T04:22:16.015 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:22:15 smithi089 ceph-mon[64076]: osdmap e165: 8 total, 8 up, 8 in 2024-09-21T04:22:16.971 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:22:16 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:22:16.971 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:22:16 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:22:16.971 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:22:16 smithi089 ceph-mon[64076]: pgmap v341: 97 pgs: 97 active+clean; 218 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 5.2 KiB/s rd, 2.1 KiB/s wr, 7 op/s 2024-09-21T04:22:16.971 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:22:16 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:22:16.972 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:22:16 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:22:17.052 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:22:16 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:22:17.053 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:22:16 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:22:17.053 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:22:16 smithi195 ceph-mon[49435]: pgmap v341: 97 pgs: 97 active+clean; 218 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 5.2 KiB/s rd, 2.1 KiB/s wr, 7 op/s 2024-09-21T04:22:17.053 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:22:16 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:22:17.053 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:22:16 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:22:17.949 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:22:17 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T04:22:17.949 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:22:17 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:22:17.949 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:22:17 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:22:18.109 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:22:17 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T04:22:18.109 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:22:17 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:22:18.109 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:22:17 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:22:18.958 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:22:18 smithi089 ceph-mon[64076]: pgmap v342: 97 pgs: 97 active+clean; 218 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 2.8 KiB/s rd, 1.3 KiB/s wr, 4 op/s 2024-09-21T04:22:19.109 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:22:18 smithi195 ceph-mon[49435]: pgmap v342: 97 pgs: 97 active+clean; 218 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 2.8 KiB/s rd, 1.3 KiB/s wr, 4 op/s 2024-09-21T04:22:21.110 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:22:20 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:22:21.110 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:22:20 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:22:21.110 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:22:20 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T04:22:21.110 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:22:20 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T04:22:21.110 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:22:20 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:22:21.110 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:22:20 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:22:21.110 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:22:20 smithi195 ceph-mon[49435]: pgmap v343: 97 pgs: 97 active+clean; 218 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 2.8 KiB/s rd, 1.2 KiB/s wr, 4 op/s 2024-09-21T04:22:21.110 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:22:20 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T04:22:21.110 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:22:20 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "versions"} : dispatch 2024-09-21T04:22:21.110 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:22:20 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "versions"} : dispatch 2024-09-21T04:22:21.110 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:22:20 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "versions"} : dispatch 2024-09-21T04:22:21.110 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:22:20 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "versions"} : dispatch 2024-09-21T04:22:21.111 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:22:20 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "versions"} : dispatch 2024-09-21T04:22:21.111 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:22:20 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "versions"} : dispatch 2024-09-21T04:22:21.111 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:22:20 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "versions"} : dispatch 2024-09-21T04:22:21.111 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:22:20 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "versions"} : dispatch 2024-09-21T04:22:21.111 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:22:20 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "versions"} : dispatch 2024-09-21T04:22:21.111 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:22:20 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "versions"} : dispatch 2024-09-21T04:22:21.111 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:22:20 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "versions"} : dispatch 2024-09-21T04:22:21.111 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:22:20 smithi195 ceph-mon[49435]: Upgrade: Setting container_image for all nfs 2024-09-21T04:22:21.111 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:22:20 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:22:21.111 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:22:20 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.0.0.smithi089.dcjmqc"} : dispatch 2024-09-21T04:22:21.111 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:22:20 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.0.0.smithi089.dcjmqc"}]': finished 2024-09-21T04:22:21.111 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:22:20 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.1.0.smithi195.vhoayp"} : dispatch 2024-09-21T04:22:21.111 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:22:20 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.1.0.smithi195.vhoayp"}]': finished 2024-09-21T04:22:21.111 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:22:20 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "versions"} : dispatch 2024-09-21T04:22:21.111 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:22:20 smithi195 ceph-mon[49435]: Upgrade: Setting container_image for all nvmeof 2024-09-21T04:22:21.112 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:22:20 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:22:21.112 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:22:20 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "versions"} : dispatch 2024-09-21T04:22:21.112 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:22:20 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "versions"} : dispatch 2024-09-21T04:22:21.112 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:22:20 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "versions"} : dispatch 2024-09-21T04:22:21.265 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:22:20 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:22:21.265 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:22:20 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:22:21.265 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:22:20 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T04:22:21.266 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:22:20 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T04:22:21.266 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:22:20 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:22:21.266 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:22:20 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:22:21.266 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:22:20 smithi089 ceph-mon[64076]: pgmap v343: 97 pgs: 97 active+clean; 218 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 2.8 KiB/s rd, 1.2 KiB/s wr, 4 op/s 2024-09-21T04:22:21.266 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:22:20 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T04:22:21.266 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:22:20 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "versions"} : dispatch 2024-09-21T04:22:21.266 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:22:20 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "versions"} : dispatch 2024-09-21T04:22:21.266 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:22:20 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "versions"} : dispatch 2024-09-21T04:22:21.266 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:22:20 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "versions"} : dispatch 2024-09-21T04:22:21.266 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:22:20 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "versions"} : dispatch 2024-09-21T04:22:21.266 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:22:20 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "versions"} : dispatch 2024-09-21T04:22:21.266 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:22:20 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "versions"} : dispatch 2024-09-21T04:22:21.266 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:22:20 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "versions"} : dispatch 2024-09-21T04:22:21.267 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:22:20 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "versions"} : dispatch 2024-09-21T04:22:21.267 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:22:20 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "versions"} : dispatch 2024-09-21T04:22:21.267 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:22:20 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "versions"} : dispatch 2024-09-21T04:22:21.267 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:22:20 smithi089 ceph-mon[64076]: Upgrade: Setting container_image for all nfs 2024-09-21T04:22:21.267 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:22:20 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:22:21.267 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:22:20 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.0.0.smithi089.dcjmqc"} : dispatch 2024-09-21T04:22:21.267 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:22:20 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.0.0.smithi089.dcjmqc"}]': finished 2024-09-21T04:22:21.267 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:22:20 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.1.0.smithi195.vhoayp"} : dispatch 2024-09-21T04:22:21.267 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:22:20 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.1.0.smithi195.vhoayp"}]': finished 2024-09-21T04:22:21.267 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:22:20 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "versions"} : dispatch 2024-09-21T04:22:21.268 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:22:20 smithi089 ceph-mon[64076]: Upgrade: Setting container_image for all nvmeof 2024-09-21T04:22:21.268 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:22:20 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:22:21.268 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:22:20 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "versions"} : dispatch 2024-09-21T04:22:21.268 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:22:20 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "versions"} : dispatch 2024-09-21T04:22:21.268 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:22:20 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "versions"} : dispatch 2024-09-21T04:22:23.515 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:22:23 smithi089 ceph-mon[64076]: Upgrade: Updating grafana.smithi089 2024-09-21T04:22:23.515 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:22:23 smithi089 ceph-mon[64076]: Deploying daemon grafana.smithi089 on smithi089 2024-09-21T04:22:23.515 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:22:23 smithi089 ceph-mon[64076]: pgmap v344: 97 pgs: 97 active+clean; 218 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 2.8 KiB/s rd, 1.2 KiB/s wr, 4 op/s 2024-09-21T04:22:23.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:22:23 smithi195 ceph-mon[49435]: Upgrade: Updating grafana.smithi089 2024-09-21T04:22:23.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:22:23 smithi195 ceph-mon[49435]: Deploying daemon grafana.smithi089 on smithi089 2024-09-21T04:22:23.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:22:23 smithi195 ceph-mon[49435]: pgmap v344: 97 pgs: 97 active+clean; 218 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 2.8 KiB/s rd, 1.2 KiB/s wr, 4 op/s 2024-09-21T04:22:25.514 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:22:25 smithi089 ceph-mon[64076]: pgmap v345: 97 pgs: 97 active+clean; 218 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 102 B/s rd, 409 B/s wr, 0 op/s 2024-09-21T04:22:25.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:22:25 smithi195 ceph-mon[49435]: pgmap v345: 97 pgs: 97 active+clean; 218 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 102 B/s rd, 409 B/s wr, 0 op/s 2024-09-21T04:22:27.515 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:22:27 smithi089 ceph-mon[64076]: pgmap v346: 97 pgs: 97 active+clean; 218 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 89 B/s rd, 358 B/s wr, 0 op/s 2024-09-21T04:22:27.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:22:27 smithi195 ceph-mon[49435]: pgmap v346: 97 pgs: 97 active+clean; 218 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 89 B/s rd, 358 B/s wr, 0 op/s 2024-09-21T04:22:29.514 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:22:29 smithi089 ceph-mon[64076]: pgmap v347: 97 pgs: 97 active+clean; 218 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 341 B/s wr, 0 op/s 2024-09-21T04:22:29.610 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:22:29 smithi195 ceph-mon[49435]: pgmap v347: 97 pgs: 97 active+clean; 218 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 341 B/s wr, 0 op/s 2024-09-21T04:22:31.458 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:22:31 smithi089 ceph-mon[64076]: pgmap v348: 97 pgs: 97 active+clean; 218 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T04:22:31.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:22:31 smithi195 ceph-mon[49435]: pgmap v348: 97 pgs: 97 active+clean; 218 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T04:22:32.309 INFO:teuthology.orchestra.run.smithi089.stdout:true 2024-09-21T04:22:33.059 INFO:teuthology.orchestra.run.smithi089.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-21T04:22:33.060 INFO:teuthology.orchestra.run.smithi089.stdout:alertmanager.smithi089 smithi089 *:9093,9094 running (10m) 17s ago 19m 22.0M - 0.27.0 11f11916f8cd 65dfcbcafee1 2024-09-21T04:22:33.060 INFO:teuthology.orchestra.run.smithi089.stdout:crash.smithi089 smithi089 running (7m) 17s ago 19m 6891k - 19.3.0-5103-gf36283e7 48ed0a2cf89d 6c415a5e173e 2024-09-21T04:22:33.060 INFO:teuthology.orchestra.run.smithi089.stdout:crash.smithi195 smithi195 running (7m) 19s ago 19m 6891k - 19.3.0-5103-gf36283e7 48ed0a2cf89d 78ce05a4e9ab 2024-09-21T04:22:33.060 INFO:teuthology.orchestra.run.smithi089.stdout:grafana.smithi089 smithi089 *:3000 running (10m) 17s ago 19m 54.9M - dad864ee21e9 ec4d6738bce7 2024-09-21T04:22:33.060 INFO:teuthology.orchestra.run.smithi089.stdout:mds.foofs.smithi089.osufws smithi089 running (58s) 17s ago 13m 33.4M - 19.3.0-5103-gf36283e7 48ed0a2cf89d 526fde771b8f 2024-09-21T04:22:33.060 INFO:teuthology.orchestra.run.smithi089.stdout:mds.foofs.smithi195.hijxre smithi195 running (39s) 19s ago 13m 13.2M - 19.3.0-5103-gf36283e7 48ed0a2cf89d 9b532cb9643e 2024-09-21T04:22:33.060 INFO:teuthology.orchestra.run.smithi089.stdout:mgr.smithi089.srlyxf smithi089 *:8443,9283,8765 running (9m) 17s ago 20m 525M - 19.3.0-5103-gf36283e7 48ed0a2cf89d 3cbeaaa88241 2024-09-21T04:22:33.060 INFO:teuthology.orchestra.run.smithi089.stdout:mgr.smithi195.cgotzm smithi195 *:8443,9283,8765 running (8m) 19s ago 19m 458M - 19.3.0-5103-gf36283e7 48ed0a2cf89d d3734574cf66 2024-09-21T04:22:33.060 INFO:teuthology.orchestra.run.smithi089.stdout:mon.smithi089 smithi089 running (8m) 17s ago 21m 65.7M 2048M 19.3.0-5103-gf36283e7 48ed0a2cf89d 7dd34c8a6f21 2024-09-21T04:22:33.060 INFO:teuthology.orchestra.run.smithi089.stdout:mon.smithi195 smithi195 running (8m) 19s ago 19m 50.2M 2048M 19.3.0-5103-gf36283e7 48ed0a2cf89d 4255937444e7 2024-09-21T04:22:33.060 INFO:teuthology.orchestra.run.smithi089.stdout:nfs.foo.0.0.smithi089.dcjmqc smithi089 *:2049 running (27s) 17s ago 13m 62.2M - 5.9 48ed0a2cf89d 9fd550a02a73 2024-09-21T04:22:33.060 INFO:teuthology.orchestra.run.smithi089.stdout:nfs.foo.1.0.smithi195.vhoayp smithi195 *:2049 running (22s) 19s ago 13m 17.8M - 5.9 48ed0a2cf89d 14ac48ce3073 2024-09-21T04:22:33.061 INFO:teuthology.orchestra.run.smithi089.stdout:node-exporter.smithi089 smithi089 *:9100 running (10m) 17s ago 19m 11.3M - 1.7.0 72c9c2088986 7ef0d1588475 2024-09-21T04:22:33.061 INFO:teuthology.orchestra.run.smithi089.stdout:node-exporter.smithi195 smithi195 *:9100 running (9m) 19s ago 18m 11.1M - 1.7.0 72c9c2088986 61eafd61bc6c 2024-09-21T04:22:33.061 INFO:teuthology.orchestra.run.smithi089.stdout:osd.0 smithi089 running (6m) 17s ago 18m 432M 1935M 19.3.0-5103-gf36283e7 48ed0a2cf89d 3df0adea59fb 2024-09-21T04:22:33.061 INFO:teuthology.orchestra.run.smithi089.stdout:osd.1 smithi089 running (5m) 17s ago 17m 329M 1935M 19.3.0-5103-gf36283e7 48ed0a2cf89d da2e49774e7e 2024-09-21T04:22:33.061 INFO:teuthology.orchestra.run.smithi089.stdout:osd.2 smithi089 running (4m) 17s ago 17m 319M 1935M 19.3.0-5103-gf36283e7 48ed0a2cf89d b9f6438dd5b9 2024-09-21T04:22:33.061 INFO:teuthology.orchestra.run.smithi089.stdout:osd.3 smithi089 running (4m) 17s ago 16m 372M 1935M 19.3.0-5103-gf36283e7 48ed0a2cf89d d269cd94bd08 2024-09-21T04:22:33.061 INFO:teuthology.orchestra.run.smithi089.stdout:osd.4 smithi195 running (3m) 19s ago 16m 467M 2704M 19.3.0-5103-gf36283e7 48ed0a2cf89d e375f90f95c4 2024-09-21T04:22:33.061 INFO:teuthology.orchestra.run.smithi089.stdout:osd.5 smithi195 running (2m) 19s ago 16m 340M 2704M 19.3.0-5103-gf36283e7 48ed0a2cf89d 41de0beb713d 2024-09-21T04:22:33.061 INFO:teuthology.orchestra.run.smithi089.stdout:osd.6 smithi195 running (2m) 19s ago 15m 285M 2704M 19.3.0-5103-gf36283e7 48ed0a2cf89d a41d0c0574ab 2024-09-21T04:22:33.061 INFO:teuthology.orchestra.run.smithi089.stdout:osd.7 smithi195 running (86s) 19s ago 15m 250M 2704M 19.3.0-5103-gf36283e7 48ed0a2cf89d 66d3bf5f6c5f 2024-09-21T04:22:33.061 INFO:teuthology.orchestra.run.smithi089.stdout:prometheus.smithi089 smithi089 *:9095 running (9m) 17s ago 18m 58.8M - 2.51.0 1d3b7f56885b 8c9f309a6459 2024-09-21T04:22:33.493 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:22:33 smithi089 ceph-mon[64076]: pgmap v349: 97 pgs: 97 active+clean; 218 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T04:22:33.494 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:22:33 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T04:22:33.494 INFO:teuthology.orchestra.run.smithi089.stdout:{ 2024-09-21T04:22:33.494 INFO:teuthology.orchestra.run.smithi089.stdout: "mon": { 2024-09-21T04:22:33.494 INFO:teuthology.orchestra.run.smithi089.stdout: "ceph version 19.3.0-5103-gf36283e7 (f36283e7c91cac1c01cb9a7c4ff21bd697dbd4f2) squid (dev)": 2 2024-09-21T04:22:33.494 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2024-09-21T04:22:33.494 INFO:teuthology.orchestra.run.smithi089.stdout: "mgr": { 2024-09-21T04:22:33.494 INFO:teuthology.orchestra.run.smithi089.stdout: "ceph version 19.3.0-5103-gf36283e7 (f36283e7c91cac1c01cb9a7c4ff21bd697dbd4f2) squid (dev)": 2 2024-09-21T04:22:33.494 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2024-09-21T04:22:33.494 INFO:teuthology.orchestra.run.smithi089.stdout: "osd": { 2024-09-21T04:22:33.495 INFO:teuthology.orchestra.run.smithi089.stdout: "ceph version 19.3.0-5103-gf36283e7 (f36283e7c91cac1c01cb9a7c4ff21bd697dbd4f2) squid (dev)": 8 2024-09-21T04:22:33.495 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2024-09-21T04:22:33.495 INFO:teuthology.orchestra.run.smithi089.stdout: "mds": { 2024-09-21T04:22:33.495 INFO:teuthology.orchestra.run.smithi089.stdout: "ceph version 19.3.0-5103-gf36283e7 (f36283e7c91cac1c01cb9a7c4ff21bd697dbd4f2) squid (dev)": 2 2024-09-21T04:22:33.495 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2024-09-21T04:22:33.495 INFO:teuthology.orchestra.run.smithi089.stdout: "overall": { 2024-09-21T04:22:33.495 INFO:teuthology.orchestra.run.smithi089.stdout: "ceph version 19.3.0-5103-gf36283e7 (f36283e7c91cac1c01cb9a7c4ff21bd697dbd4f2) squid (dev)": 14 2024-09-21T04:22:33.495 INFO:teuthology.orchestra.run.smithi089.stdout: } 2024-09-21T04:22:33.495 INFO:teuthology.orchestra.run.smithi089.stdout:} 2024-09-21T04:22:33.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:22:33 smithi195 ceph-mon[49435]: pgmap v349: 97 pgs: 97 active+clean; 218 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T04:22:33.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:22:33 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T04:22:33.853 INFO:teuthology.orchestra.run.smithi089.stdout:{ 2024-09-21T04:22:33.853 INFO:teuthology.orchestra.run.smithi089.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:2195e5f178be0a9390728e2a14088236d8fe6f82635f5c84418fe37a04457e00", 2024-09-21T04:22:33.853 INFO:teuthology.orchestra.run.smithi089.stdout: "in_progress": true, 2024-09-21T04:22:33.853 INFO:teuthology.orchestra.run.smithi089.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-21T04:22:33.853 INFO:teuthology.orchestra.run.smithi089.stdout: "services_complete": [ 2024-09-21T04:22:33.853 INFO:teuthology.orchestra.run.smithi089.stdout: "osd", 2024-09-21T04:22:33.853 INFO:teuthology.orchestra.run.smithi089.stdout: "crash", 2024-09-21T04:22:33.853 INFO:teuthology.orchestra.run.smithi089.stdout: "nfs", 2024-09-21T04:22:33.853 INFO:teuthology.orchestra.run.smithi089.stdout: "mgr", 2024-09-21T04:22:33.853 INFO:teuthology.orchestra.run.smithi089.stdout: "mds", 2024-09-21T04:22:33.853 INFO:teuthology.orchestra.run.smithi089.stdout: "mon" 2024-09-21T04:22:33.854 INFO:teuthology.orchestra.run.smithi089.stdout: ], 2024-09-21T04:22:33.854 INFO:teuthology.orchestra.run.smithi089.stdout: "progress": "18/23 daemons upgraded", 2024-09-21T04:22:33.854 INFO:teuthology.orchestra.run.smithi089.stdout: "message": "Currently upgrading grafana daemons", 2024-09-21T04:22:33.854 INFO:teuthology.orchestra.run.smithi089.stdout: "is_paused": false 2024-09-21T04:22:33.854 INFO:teuthology.orchestra.run.smithi089.stdout:} 2024-09-21T04:22:34.295 INFO:teuthology.orchestra.run.smithi089.stdout:HEALTH_OK 2024-09-21T04:22:34.515 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:22:34 smithi089 ceph-mon[64076]: from='client.34568 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T04:22:34.515 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:22:34 smithi089 ceph-mon[64076]: from='client.34572 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T04:22:34.515 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:22:34 smithi089 ceph-mon[64076]: from='client.34576 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T04:22:34.515 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:22:34 smithi089 ceph-mon[64076]: from='client.? 172.21.15.89:0/1505786724' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T04:22:34.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:22:34 smithi195 ceph-mon[49435]: from='client.34568 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T04:22:34.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:22:34 smithi195 ceph-mon[49435]: from='client.34572 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T04:22:34.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:22:34 smithi195 ceph-mon[49435]: from='client.34576 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T04:22:34.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:22:34 smithi195 ceph-mon[49435]: from='client.? 172.21.15.89:0/1505786724' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T04:22:35.515 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:22:35 smithi089 ceph-mon[64076]: from='client.34584 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T04:22:35.515 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:22:35 smithi089 ceph-mon[64076]: pgmap v350: 97 pgs: 97 active+clean; 218 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T04:22:35.515 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:22:35 smithi089 ceph-mon[64076]: from='client.? 172.21.15.89:0/978854057' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-21T04:22:35.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:22:35 smithi195 ceph-mon[49435]: from='client.34584 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T04:22:35.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:22:35 smithi195 ceph-mon[49435]: pgmap v350: 97 pgs: 97 active+clean; 218 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T04:22:35.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:22:35 smithi195 ceph-mon[49435]: from='client.? 172.21.15.89:0/978854057' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-21T04:22:37.265 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:22:37 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:22:37.265 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:22:37 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:22:37.265 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:22:37 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T04:22:37.265 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:22:37 smithi089 ceph-mon[64076]: pgmap v351: 97 pgs: 97 active+clean; 218 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T04:22:37.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:22:37 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:22:37.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:22:37 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:22:37.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:22:37 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T04:22:37.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:22:37 smithi195 ceph-mon[49435]: pgmap v351: 97 pgs: 97 active+clean; 218 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T04:22:39.265 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:22:39 smithi089 ceph-mon[64076]: pgmap v352: 97 pgs: 97 active+clean; 218 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-21T04:22:39.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:22:39 smithi195 ceph-mon[49435]: pgmap v352: 97 pgs: 97 active+clean; 218 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-21T04:22:41.515 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:22:41 smithi089 ceph-mon[64076]: pgmap v353: 97 pgs: 97 active+clean; 218 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T04:22:41.515 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:22:41 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:22:41.515 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:22:41 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:22:41.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:22:41 smithi195 ceph-mon[49435]: pgmap v353: 97 pgs: 97 active+clean; 218 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T04:22:41.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:22:41 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:22:41.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:22:41 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:22:42.766 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:22:42 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:22:42.766 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:22:42 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:22:42.766 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:22:42 smithi089 ceph-mon[64076]: pgmap v354: 97 pgs: 97 active+clean; 218 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T04:22:42.859 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:22:42 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:22:42.859 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:22:42 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:22:42.859 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:22:42 smithi195 ceph-mon[49435]: pgmap v354: 97 pgs: 97 active+clean; 218 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T04:22:45.515 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:22:45 smithi089 ceph-mon[64076]: pgmap v355: 97 pgs: 97 active+clean; 218 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T04:22:45.515 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:22:45 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:22:45.515 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:22:45 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:22:45.515 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:22:45 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T04:22:45.516 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:22:45 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T04:22:45.516 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:22:45 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:22:45.516 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:22:45 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:22:45.516 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:22:45 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "dashboard get-grafana-api-url"} : dispatch 2024-09-21T04:22:45.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:22:45 smithi195 ceph-mon[49435]: pgmap v355: 97 pgs: 97 active+clean; 218 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T04:22:45.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:22:45 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:22:45.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:22:45 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:22:45.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:22:45 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T04:22:45.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:22:45 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T04:22:45.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:22:45 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:22:45.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:22:45 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:22:45.610 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:22:45 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "dashboard get-grafana-api-url"} : dispatch 2024-09-21T04:22:46.515 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:22:46 smithi089 ceph-mon[64076]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-21T04:22:46.515 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:22:46 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T04:22:46.515 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:22:46 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "versions"} : dispatch 2024-09-21T04:22:46.515 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:22:46 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "versions"} : dispatch 2024-09-21T04:22:46.515 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:22:46 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "versions"} : dispatch 2024-09-21T04:22:46.515 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:22:46 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "versions"} : dispatch 2024-09-21T04:22:46.515 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:22:46 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "versions"} : dispatch 2024-09-21T04:22:46.516 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:22:46 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "versions"} : dispatch 2024-09-21T04:22:46.516 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:22:46 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "versions"} : dispatch 2024-09-21T04:22:46.516 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:22:46 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "versions"} : dispatch 2024-09-21T04:22:46.516 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:22:46 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "versions"} : dispatch 2024-09-21T04:22:46.516 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:22:46 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "versions"} : dispatch 2024-09-21T04:22:46.516 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:22:46 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "versions"} : dispatch 2024-09-21T04:22:46.516 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:22:46 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "versions"} : dispatch 2024-09-21T04:22:46.516 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:22:46 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "versions"} : dispatch 2024-09-21T04:22:46.516 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:22:46 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "versions"} : dispatch 2024-09-21T04:22:46.516 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:22:46 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "versions"} : dispatch 2024-09-21T04:22:46.516 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:22:46 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "versions"} : dispatch 2024-09-21T04:22:46.516 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:22:46 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "versions"} : dispatch 2024-09-21T04:22:46.516 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:22:46 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "versions"} : dispatch 2024-09-21T04:22:46.516 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:22:46 smithi089 ceph-mon[64076]: Upgrade: Finalizing container_image settings 2024-09-21T04:22:46.516 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:22:46 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:22:46.516 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:22:46 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "config rm", "name": "container_image", "who": "mgr"} : dispatch 2024-09-21T04:22:46.517 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:22:46 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-09-21T04:22:46.517 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:22:46 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T04:22:46.517 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:22:46 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-09-21T04:22:46.517 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:22:46 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "config rm", "name": "container_image", "who": "client.crash"} : dispatch 2024-09-21T04:22:46.517 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:22:46 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-09-21T04:22:46.517 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:22:46 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "config rm", "name": "container_image", "who": "osd"} : dispatch 2024-09-21T04:22:46.517 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:22:46 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd"}]': finished 2024-09-21T04:22:46.517 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:22:46 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "config rm", "name": "container_image", "who": "mds"} : dispatch 2024-09-21T04:22:46.517 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:22:46 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-09-21T04:22:46.517 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:22:46 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rgw"} : dispatch 2024-09-21T04:22:46.517 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:22:46 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]': finished 2024-09-21T04:22:46.517 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:22:46 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"} : dispatch 2024-09-21T04:22:46.517 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:22:46 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-09-21T04:22:46.517 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:22:46 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T04:22:46.518 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:22:46 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"} : dispatch 2024-09-21T04:22:46.518 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:22:46 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-09-21T04:22:46.518 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:22:46 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "config rm", "name": "container_image", "who": "client.iscsi"} : dispatch 2024-09-21T04:22:46.518 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:22:46 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]': finished 2024-09-21T04:22:46.518 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:22:46 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nfs"} : dispatch 2024-09-21T04:22:46.518 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:22:46 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-09-21T04:22:46.518 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:22:46 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"} : dispatch 2024-09-21T04:22:46.518 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:22:46 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-09-21T04:22:46.518 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:22:46 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T04:22:46.518 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:22:46 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T04:22:46.518 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:22:46 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T04:22:46.519 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:22:46 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T04:22:46.519 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:22:46 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T04:22:46.519 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:22:46 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T04:22:46.519 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:22:46 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"} : dispatch 2024-09-21T04:22:46.519 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:22:46 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-09-21T04:22:46.519 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:22:46 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T04:22:46.519 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:22:46 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:22:46.519 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:22:46 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:22:46.519 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:22:46 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T04:22:46.519 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:22:46 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T04:22:46.519 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:22:46 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:22:46.519 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:22:46 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:22:46.519 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:22:46 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T04:22:46.519 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:22:46 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T04:22:46.519 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:22:46 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T04:22:46.520 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:22:46 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:22:46.520 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:22:46 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:22:46.610 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:22:46 smithi195 ceph-mon[49435]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-21T04:22:46.610 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:22:46 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T04:22:46.610 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:22:46 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "versions"} : dispatch 2024-09-21T04:22:46.610 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:22:46 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "versions"} : dispatch 2024-09-21T04:22:46.610 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:22:46 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "versions"} : dispatch 2024-09-21T04:22:46.610 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:22:46 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "versions"} : dispatch 2024-09-21T04:22:46.610 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:22:46 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "versions"} : dispatch 2024-09-21T04:22:46.610 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:22:46 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "versions"} : dispatch 2024-09-21T04:22:46.610 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:22:46 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "versions"} : dispatch 2024-09-21T04:22:46.610 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:22:46 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "versions"} : dispatch 2024-09-21T04:22:46.610 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:22:46 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "versions"} : dispatch 2024-09-21T04:22:46.610 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:22:46 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "versions"} : dispatch 2024-09-21T04:22:46.611 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:22:46 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "versions"} : dispatch 2024-09-21T04:22:46.611 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:22:46 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "versions"} : dispatch 2024-09-21T04:22:46.611 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:22:46 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "versions"} : dispatch 2024-09-21T04:22:46.611 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:22:46 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "versions"} : dispatch 2024-09-21T04:22:46.611 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:22:46 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "versions"} : dispatch 2024-09-21T04:22:46.611 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:22:46 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "versions"} : dispatch 2024-09-21T04:22:46.611 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:22:46 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "versions"} : dispatch 2024-09-21T04:22:46.611 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:22:46 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "versions"} : dispatch 2024-09-21T04:22:46.611 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:22:46 smithi195 ceph-mon[49435]: Upgrade: Finalizing container_image settings 2024-09-21T04:22:46.611 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:22:46 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:22:46.611 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:22:46 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "config rm", "name": "container_image", "who": "mgr"} : dispatch 2024-09-21T04:22:46.611 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:22:46 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-09-21T04:22:46.611 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:22:46 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T04:22:46.611 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:22:46 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-09-21T04:22:46.611 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:22:46 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "config rm", "name": "container_image", "who": "client.crash"} : dispatch 2024-09-21T04:22:46.612 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:22:46 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-09-21T04:22:46.612 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:22:46 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "config rm", "name": "container_image", "who": "osd"} : dispatch 2024-09-21T04:22:46.612 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:22:46 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd"}]': finished 2024-09-21T04:22:46.612 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:22:46 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "config rm", "name": "container_image", "who": "mds"} : dispatch 2024-09-21T04:22:46.612 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:22:46 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-09-21T04:22:46.612 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:22:46 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rgw"} : dispatch 2024-09-21T04:22:46.612 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:22:46 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]': finished 2024-09-21T04:22:46.612 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:22:46 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"} : dispatch 2024-09-21T04:22:46.612 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:22:46 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-09-21T04:22:46.612 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:22:46 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T04:22:46.612 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:22:46 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"} : dispatch 2024-09-21T04:22:46.612 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:22:46 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-09-21T04:22:46.612 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:22:46 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "config rm", "name": "container_image", "who": "client.iscsi"} : dispatch 2024-09-21T04:22:46.612 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:22:46 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]': finished 2024-09-21T04:22:46.612 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:22:46 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nfs"} : dispatch 2024-09-21T04:22:46.613 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:22:46 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-09-21T04:22:46.613 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:22:46 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"} : dispatch 2024-09-21T04:22:46.613 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:22:46 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-09-21T04:22:46.613 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:22:46 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T04:22:46.613 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:22:46 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T04:22:46.613 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:22:46 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T04:22:46.613 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:22:46 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T04:22:46.613 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:22:46 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T04:22:46.613 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:22:46 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T04:22:46.613 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:22:46 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"} : dispatch 2024-09-21T04:22:46.613 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:22:46 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-09-21T04:22:46.613 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:22:46 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T04:22:46.613 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:22:46 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:22:46.613 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:22:46 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:22:46.614 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:22:46 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T04:22:46.614 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:22:46 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T04:22:46.614 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:22:46 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:22:46.614 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:22:46 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:22:46.614 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:22:46 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T04:22:46.614 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:22:46 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T04:22:46.614 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:22:46 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T04:22:46.614 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:22:46 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:22:46.614 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:22:46 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:22:47.515 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:22:47 smithi089 ceph-mon[64076]: Upgrade: Complete! 2024-09-21T04:22:47.515 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:22:47 smithi089 ceph-mon[64076]: pgmap v356: 97 pgs: 97 active+clean; 218 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T04:22:47.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:22:47 smithi195 ceph-mon[49435]: Upgrade: Complete! 2024-09-21T04:22:47.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:22:47 smithi195 ceph-mon[49435]: pgmap v356: 97 pgs: 97 active+clean; 218 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T04:22:48.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:22:48 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T04:22:48.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:22:48 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:22:48.515 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:22:48 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T04:22:48.515 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:22:48 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:22:49.515 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:22:49 smithi089 ceph-mon[64076]: pgmap v357: 97 pgs: 97 active+clean; 218 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-21T04:22:49.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:22:49 smithi195 ceph-mon[49435]: pgmap v357: 97 pgs: 97 active+clean; 218 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-21T04:22:51.515 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:22:51 smithi089 ceph-mon[64076]: pgmap v358: 97 pgs: 97 active+clean; 218 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T04:22:51.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:22:51 smithi195 ceph-mon[49435]: pgmap v358: 97 pgs: 97 active+clean; 218 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T04:22:53.515 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:22:53 smithi089 ceph-mon[64076]: pgmap v359: 97 pgs: 97 active+clean; 218 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T04:22:53.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:22:53 smithi195 ceph-mon[49435]: pgmap v359: 97 pgs: 97 active+clean; 218 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T04:22:55.515 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:22:55 smithi089 ceph-mon[64076]: pgmap v360: 97 pgs: 97 active+clean; 218 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T04:22:55.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:22:55 smithi195 ceph-mon[49435]: pgmap v360: 97 pgs: 97 active+clean; 218 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T04:22:57.515 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:22:57 smithi089 ceph-mon[64076]: pgmap v361: 97 pgs: 97 active+clean; 218 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T04:22:57.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:22:57 smithi195 ceph-mon[49435]: pgmap v361: 97 pgs: 97 active+clean; 218 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T04:22:59.515 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:22:59 smithi089 ceph-mon[64076]: pgmap v362: 97 pgs: 97 active+clean; 218 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-21T04:22:59.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:22:59 smithi195 ceph-mon[49435]: pgmap v362: 97 pgs: 97 active+clean; 218 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-21T04:23:01.515 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:23:01 smithi089 ceph-mon[64076]: pgmap v363: 97 pgs: 97 active+clean; 218 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T04:23:01.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:23:01 smithi195 ceph-mon[49435]: pgmap v363: 97 pgs: 97 active+clean; 218 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T04:23:03.515 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:23:03 smithi089 ceph-mon[64076]: pgmap v364: 97 pgs: 97 active+clean; 218 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T04:23:03.515 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:23:03 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T04:23:03.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:23:03 smithi195 ceph-mon[49435]: pgmap v364: 97 pgs: 97 active+clean; 218 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T04:23:03.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:23:03 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T04:23:05.513 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:23:05 smithi089 ceph-mon[64076]: pgmap v365: 97 pgs: 97 active+clean; 218 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T04:23:05.535 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 1b9b79fc-77ce-11ef-baf5-efdc52797490 -e sha1=f36283e7c91cac1c01cb9a7c4ff21bd697dbd4f2 -- bash -c 'ceph orch ps' 2024-09-21T04:23:05.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:23:05 smithi195 ceph-mon[49435]: pgmap v365: 97 pgs: 97 active+clean; 218 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T04:23:06.515 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:23:06 smithi089 ceph-mon[64076]: from='client.44409 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T04:23:06.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:23:06 smithi195 ceph-mon[49435]: from='client.44409 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T04:23:07.207 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:23:07 smithi089 ceph-mon[64076]: pgmap v366: 97 pgs: 97 active+clean; 218 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T04:23:07.510 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:23:07 smithi195 ceph-mon[49435]: pgmap v366: 97 pgs: 97 active+clean; 218 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T04:23:07.890 INFO:teuthology.orchestra.run.smithi089.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-21T04:23:07.891 INFO:teuthology.orchestra.run.smithi089.stdout:alertmanager.smithi089 smithi089 *:9093,9094 running (11m) 27s ago 20m 22.0M - 0.27.0 11f11916f8cd 65dfcbcafee1 2024-09-21T04:23:07.891 INFO:teuthology.orchestra.run.smithi089.stdout:crash.smithi089 smithi089 running (7m) 27s ago 20m 6891k - 19.3.0-5103-gf36283e7 48ed0a2cf89d 6c415a5e173e 2024-09-21T04:23:07.891 INFO:teuthology.orchestra.run.smithi089.stdout:crash.smithi195 smithi195 running (7m) 54s ago 19m 6891k - 19.3.0-5103-gf36283e7 48ed0a2cf89d 78ce05a4e9ab 2024-09-21T04:23:07.891 INFO:teuthology.orchestra.run.smithi089.stdout:grafana.smithi089 smithi089 *:3000 running (35s) 27s ago 20m 43.0M - 10.4.8 5aad1d7cf004 f8eb2943066e 2024-09-21T04:23:07.891 INFO:teuthology.orchestra.run.smithi089.stdout:mds.foofs.smithi089.osufws smithi089 running (93s) 27s ago 14m 33.6M - 19.3.0-5103-gf36283e7 48ed0a2cf89d 526fde771b8f 2024-09-21T04:23:07.891 INFO:teuthology.orchestra.run.smithi089.stdout:mds.foofs.smithi195.hijxre smithi195 running (74s) 54s ago 14m 13.2M - 19.3.0-5103-gf36283e7 48ed0a2cf89d 9b532cb9643e 2024-09-21T04:23:07.892 INFO:teuthology.orchestra.run.smithi089.stdout:mgr.smithi089.srlyxf smithi089 *:8443,9283,8765 running (10m) 27s ago 21m 533M - 19.3.0-5103-gf36283e7 48ed0a2cf89d 3cbeaaa88241 2024-09-21T04:23:07.892 INFO:teuthology.orchestra.run.smithi089.stdout:mgr.smithi195.cgotzm smithi195 *:8443,9283,8765 running (9m) 54s ago 19m 458M - 19.3.0-5103-gf36283e7 48ed0a2cf89d d3734574cf66 2024-09-21T04:23:07.892 INFO:teuthology.orchestra.run.smithi089.stdout:mon.smithi089 smithi089 running (9m) 27s ago 21m 66.3M 2048M 19.3.0-5103-gf36283e7 48ed0a2cf89d 7dd34c8a6f21 2024-09-21T04:23:07.892 INFO:teuthology.orchestra.run.smithi089.stdout:mon.smithi195 smithi195 running (8m) 54s ago 19m 50.2M 2048M 19.3.0-5103-gf36283e7 48ed0a2cf89d 4255937444e7 2024-09-21T04:23:07.892 INFO:teuthology.orchestra.run.smithi089.stdout:nfs.foo.0.0.smithi089.dcjmqc smithi089 *:2049 running (62s) 27s ago 13m 63.1M - 5.9 48ed0a2cf89d 9fd550a02a73 2024-09-21T04:23:07.892 INFO:teuthology.orchestra.run.smithi089.stdout:nfs.foo.1.0.smithi195.vhoayp smithi195 *:2049 running (57s) 54s ago 13m 17.8M - 5.9 48ed0a2cf89d 14ac48ce3073 2024-09-21T04:23:07.892 INFO:teuthology.orchestra.run.smithi089.stdout:node-exporter.smithi089 smithi089 *:9100 running (11m) 27s ago 20m 10.8M - 1.7.0 72c9c2088986 7ef0d1588475 2024-09-21T04:23:07.893 INFO:teuthology.orchestra.run.smithi089.stdout:node-exporter.smithi195 smithi195 *:9100 running (10m) 54s ago 19m 11.1M - 1.7.0 72c9c2088986 61eafd61bc6c 2024-09-21T04:23:07.893 INFO:teuthology.orchestra.run.smithi089.stdout:osd.0 smithi089 running (7m) 27s ago 18m 434M 1935M 19.3.0-5103-gf36283e7 48ed0a2cf89d 3df0adea59fb 2024-09-21T04:23:07.893 INFO:teuthology.orchestra.run.smithi089.stdout:osd.1 smithi089 running (6m) 27s ago 18m 329M 1935M 19.3.0-5103-gf36283e7 48ed0a2cf89d da2e49774e7e 2024-09-21T04:23:07.893 INFO:teuthology.orchestra.run.smithi089.stdout:osd.2 smithi089 running (5m) 27s ago 17m 319M 1935M 19.3.0-5103-gf36283e7 48ed0a2cf89d b9f6438dd5b9 2024-09-21T04:23:07.893 INFO:teuthology.orchestra.run.smithi089.stdout:osd.3 smithi089 running (4m) 27s ago 17m 373M 1935M 19.3.0-5103-gf36283e7 48ed0a2cf89d d269cd94bd08 2024-09-21T04:23:07.893 INFO:teuthology.orchestra.run.smithi089.stdout:osd.4 smithi195 running (3m) 54s ago 17m 467M 2704M 19.3.0-5103-gf36283e7 48ed0a2cf89d e375f90f95c4 2024-09-21T04:23:07.893 INFO:teuthology.orchestra.run.smithi089.stdout:osd.5 smithi195 running (3m) 54s ago 16m 340M 2704M 19.3.0-5103-gf36283e7 48ed0a2cf89d 41de0beb713d 2024-09-21T04:23:07.894 INFO:teuthology.orchestra.run.smithi089.stdout:osd.6 smithi195 running (2m) 54s ago 16m 285M 2704M 19.3.0-5103-gf36283e7 48ed0a2cf89d a41d0c0574ab 2024-09-21T04:23:07.894 INFO:teuthology.orchestra.run.smithi089.stdout:osd.7 smithi195 running (2m) 54s ago 15m 250M 2704M 19.3.0-5103-gf36283e7 48ed0a2cf89d 66d3bf5f6c5f 2024-09-21T04:23:07.894 INFO:teuthology.orchestra.run.smithi089.stdout:prometheus.smithi089 smithi089 *:9095 running (9m) 27s ago 19m 58.8M - 2.51.0 1d3b7f56885b 8c9f309a6459 2024-09-21T04:23:08.604 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 1b9b79fc-77ce-11ef-baf5-efdc52797490 -e sha1=f36283e7c91cac1c01cb9a7c4ff21bd697dbd4f2 -- bash -c 'ceph versions' 2024-09-21T04:23:09.487 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:23:09 smithi089 ceph-mon[64076]: from='client.34592 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T04:23:09.487 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:23:09 smithi089 ceph-mon[64076]: pgmap v367: 97 pgs: 97 active+clean; 218 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-21T04:23:09.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:23:09 smithi195 ceph-mon[49435]: from='client.34592 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T04:23:09.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:23:09 smithi195 ceph-mon[49435]: pgmap v367: 97 pgs: 97 active+clean; 218 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-21T04:23:10.710 INFO:teuthology.orchestra.run.smithi089.stdout:{ 2024-09-21T04:23:10.710 INFO:teuthology.orchestra.run.smithi089.stdout: "mon": { 2024-09-21T04:23:10.711 INFO:teuthology.orchestra.run.smithi089.stdout: "ceph version 19.3.0-5103-gf36283e7 (f36283e7c91cac1c01cb9a7c4ff21bd697dbd4f2) squid (dev)": 2 2024-09-21T04:23:10.711 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2024-09-21T04:23:10.711 INFO:teuthology.orchestra.run.smithi089.stdout: "mgr": { 2024-09-21T04:23:10.711 INFO:teuthology.orchestra.run.smithi089.stdout: "ceph version 19.3.0-5103-gf36283e7 (f36283e7c91cac1c01cb9a7c4ff21bd697dbd4f2) squid (dev)": 2 2024-09-21T04:23:10.711 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2024-09-21T04:23:10.711 INFO:teuthology.orchestra.run.smithi089.stdout: "osd": { 2024-09-21T04:23:10.711 INFO:teuthology.orchestra.run.smithi089.stdout: "ceph version 19.3.0-5103-gf36283e7 (f36283e7c91cac1c01cb9a7c4ff21bd697dbd4f2) squid (dev)": 8 2024-09-21T04:23:10.711 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2024-09-21T04:23:10.711 INFO:teuthology.orchestra.run.smithi089.stdout: "mds": { 2024-09-21T04:23:10.711 INFO:teuthology.orchestra.run.smithi089.stdout: "ceph version 19.3.0-5103-gf36283e7 (f36283e7c91cac1c01cb9a7c4ff21bd697dbd4f2) squid (dev)": 2 2024-09-21T04:23:10.711 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2024-09-21T04:23:10.711 INFO:teuthology.orchestra.run.smithi089.stdout: "overall": { 2024-09-21T04:23:10.711 INFO:teuthology.orchestra.run.smithi089.stdout: "ceph version 19.3.0-5103-gf36283e7 (f36283e7c91cac1c01cb9a7c4ff21bd697dbd4f2) squid (dev)": 14 2024-09-21T04:23:10.711 INFO:teuthology.orchestra.run.smithi089.stdout: } 2024-09-21T04:23:10.711 INFO:teuthology.orchestra.run.smithi089.stdout:} 2024-09-21T04:23:11.330 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:23:11 smithi089 ceph-mon[64076]: pgmap v368: 97 pgs: 97 active+clean; 218 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T04:23:11.330 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:23:11 smithi089 ceph-mon[64076]: from='client.? 172.21.15.89:0/2116555060' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T04:23:11.355 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 1b9b79fc-77ce-11ef-baf5-efdc52797490 -e sha1=f36283e7c91cac1c01cb9a7c4ff21bd697dbd4f2 -- bash -c 'echo "wait for servicemap items w/ changing names to refresh"' 2024-09-21T04:23:11.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:23:11 smithi195 ceph-mon[49435]: pgmap v368: 97 pgs: 97 active+clean; 218 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T04:23:11.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:23:11 smithi195 ceph-mon[49435]: from='client.? 172.21.15.89:0/2116555060' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T04:23:13.199 INFO:teuthology.orchestra.run.smithi089.stdout:wait for servicemap items w/ changing names to refresh 2024-09-21T04:23:13.265 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:23:13 smithi089 ceph-mon[64076]: pgmap v369: 97 pgs: 97 active+clean; 218 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T04:23:13.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:23:13 smithi195 ceph-mon[49435]: pgmap v369: 97 pgs: 97 active+clean; 218 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T04:23:13.860 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 1b9b79fc-77ce-11ef-baf5-efdc52797490 -e sha1=f36283e7c91cac1c01cb9a7c4ff21bd697dbd4f2 -- bash -c 'sleep 60' 2024-09-21T04:23:15.417 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:23:15 smithi089 ceph-mon[64076]: pgmap v370: 97 pgs: 97 active+clean; 218 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T04:23:15.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:23:15 smithi195 ceph-mon[49435]: pgmap v370: 97 pgs: 97 active+clean; 218 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T04:23:17.515 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:23:17 smithi089 ceph-mon[64076]: pgmap v371: 97 pgs: 97 active+clean; 218 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T04:23:17.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:23:17 smithi195 ceph-mon[49435]: pgmap v371: 97 pgs: 97 active+clean; 218 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T04:23:18.515 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:23:18 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T04:23:18.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:23:18 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T04:23:19.515 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:23:19 smithi089 ceph-mon[64076]: pgmap v372: 97 pgs: 97 active+clean; 218 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-21T04:23:19.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:23:19 smithi195 ceph-mon[49435]: pgmap v372: 97 pgs: 97 active+clean; 218 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-21T04:23:21.515 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:23:21 smithi089 ceph-mon[64076]: pgmap v373: 97 pgs: 97 active+clean; 218 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T04:23:21.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:23:21 smithi195 ceph-mon[49435]: pgmap v373: 97 pgs: 97 active+clean; 218 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T04:23:23.515 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:23:23 smithi089 ceph-mon[64076]: pgmap v374: 97 pgs: 97 active+clean; 218 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T04:23:23.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:23:23 smithi195 ceph-mon[49435]: pgmap v374: 97 pgs: 97 active+clean; 218 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T04:23:25.515 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:23:25 smithi089 ceph-mon[64076]: pgmap v375: 97 pgs: 97 active+clean; 218 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T04:23:25.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:23:25 smithi195 ceph-mon[49435]: pgmap v375: 97 pgs: 97 active+clean; 218 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T04:23:27.515 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:23:27 smithi089 ceph-mon[64076]: pgmap v376: 97 pgs: 97 active+clean; 218 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T04:23:27.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:23:27 smithi195 ceph-mon[49435]: pgmap v376: 97 pgs: 97 active+clean; 218 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T04:23:29.515 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:23:29 smithi089 ceph-mon[64076]: pgmap v377: 97 pgs: 97 active+clean; 218 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-21T04:23:29.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:23:29 smithi195 ceph-mon[49435]: pgmap v377: 97 pgs: 97 active+clean; 218 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-21T04:23:31.515 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:23:31 smithi089 ceph-mon[64076]: pgmap v378: 97 pgs: 97 active+clean; 218 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T04:23:31.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:23:31 smithi195 ceph-mon[49435]: pgmap v378: 97 pgs: 97 active+clean; 218 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T04:23:33.515 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:23:33 smithi089 ceph-mon[64076]: pgmap v379: 97 pgs: 97 active+clean; 218 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T04:23:33.515 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:23:33 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T04:23:33.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:23:33 smithi195 ceph-mon[49435]: pgmap v379: 97 pgs: 97 active+clean; 218 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T04:23:33.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:23:33 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T04:23:35.515 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:23:35 smithi089 ceph-mon[64076]: pgmap v380: 97 pgs: 97 active+clean; 218 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T04:23:35.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:23:35 smithi195 ceph-mon[49435]: pgmap v380: 97 pgs: 97 active+clean; 218 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T04:23:37.515 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:23:37 smithi089 ceph-mon[64076]: pgmap v381: 97 pgs: 97 active+clean; 218 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T04:23:37.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:23:37 smithi195 ceph-mon[49435]: pgmap v381: 97 pgs: 97 active+clean; 218 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T04:23:39.515 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:23:39 smithi089 ceph-mon[64076]: pgmap v382: 97 pgs: 97 active+clean; 218 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-21T04:23:39.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:23:39 smithi195 ceph-mon[49435]: pgmap v382: 97 pgs: 97 active+clean; 218 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-21T04:23:41.515 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:23:41 smithi089 ceph-mon[64076]: pgmap v383: 97 pgs: 97 active+clean; 218 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T04:23:41.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:23:41 smithi195 ceph-mon[49435]: pgmap v383: 97 pgs: 97 active+clean; 218 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T04:23:43.515 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:23:43 smithi089 ceph-mon[64076]: pgmap v384: 97 pgs: 97 active+clean; 218 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T04:23:43.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:23:43 smithi195 ceph-mon[49435]: pgmap v384: 97 pgs: 97 active+clean; 218 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T04:23:45.515 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:23:45 smithi089 ceph-mon[64076]: pgmap v385: 97 pgs: 97 active+clean; 218 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T04:23:45.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:23:45 smithi195 ceph-mon[49435]: pgmap v385: 97 pgs: 97 active+clean; 218 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T04:23:47.515 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:23:47 smithi089 ceph-mon[64076]: pgmap v386: 97 pgs: 97 active+clean; 218 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T04:23:47.515 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:23:47 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T04:23:47.515 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:23:47 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:23:47.515 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:23:47 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:23:47.515 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:23:47 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T04:23:47.515 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:23:47 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T04:23:47.515 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:23:47 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:23:47.515 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:23:47 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:23:47.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:23:47 smithi195 ceph-mon[49435]: pgmap v386: 97 pgs: 97 active+clean; 218 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T04:23:47.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:23:47 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T04:23:47.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:23:47 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:23:47.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:23:47 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:23:47.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:23:47 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T04:23:47.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:23:47 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T04:23:47.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:23:47 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:23:47.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:23:47 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' 2024-09-21T04:23:48.515 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:23:48 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T04:23:48.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:23:48 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T04:23:49.515 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:23:49 smithi089 ceph-mon[64076]: pgmap v387: 97 pgs: 97 active+clean; 218 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-21T04:23:49.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:23:49 smithi195 ceph-mon[49435]: pgmap v387: 97 pgs: 97 active+clean; 218 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-21T04:23:51.515 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:23:51 smithi089 ceph-mon[64076]: pgmap v388: 97 pgs: 97 active+clean; 218 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T04:23:51.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:23:51 smithi195 ceph-mon[49435]: pgmap v388: 97 pgs: 97 active+clean; 218 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T04:23:53.515 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:23:53 smithi089 ceph-mon[64076]: pgmap v389: 97 pgs: 97 active+clean; 218 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T04:23:53.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:23:53 smithi195 ceph-mon[49435]: pgmap v389: 97 pgs: 97 active+clean; 218 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T04:23:55.515 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:23:55 smithi089 ceph-mon[64076]: pgmap v390: 97 pgs: 97 active+clean; 218 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T04:23:55.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:23:55 smithi195 ceph-mon[49435]: pgmap v390: 97 pgs: 97 active+clean; 218 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T04:23:57.515 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:23:57 smithi089 ceph-mon[64076]: pgmap v391: 97 pgs: 97 active+clean; 218 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T04:23:57.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:23:57 smithi195 ceph-mon[49435]: pgmap v391: 97 pgs: 97 active+clean; 218 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T04:23:59.515 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:23:59 smithi089 ceph-mon[64076]: pgmap v392: 97 pgs: 97 active+clean; 218 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-21T04:23:59.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:23:59 smithi195 ceph-mon[49435]: pgmap v392: 97 pgs: 97 active+clean; 218 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-21T04:24:00.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:24:00 smithi195 ceph-mon[49435]: pgmap v393: 97 pgs: 97 active+clean; 218 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T04:24:00.765 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:24:00 smithi089 ceph-mon[64076]: pgmap v393: 97 pgs: 97 active+clean; 218 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T04:24:03.515 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:24:03 smithi089 ceph-mon[64076]: pgmap v394: 97 pgs: 97 active+clean; 218 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T04:24:03.515 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:24:03 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T04:24:03.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:24:03 smithi195 ceph-mon[49435]: pgmap v394: 97 pgs: 97 active+clean; 218 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T04:24:03.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:24:03 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T04:24:05.515 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:24:05 smithi089 ceph-mon[64076]: pgmap v395: 97 pgs: 97 active+clean; 218 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T04:24:05.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:24:05 smithi195 ceph-mon[49435]: pgmap v395: 97 pgs: 97 active+clean; 218 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T04:24:07.411 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:24:07 smithi195 ceph-mon[49435]: pgmap v396: 97 pgs: 97 active+clean; 218 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T04:24:07.515 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:24:07 smithi089 ceph-mon[64076]: pgmap v396: 97 pgs: 97 active+clean; 218 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T04:24:09.515 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:24:09 smithi089 ceph-mon[64076]: pgmap v397: 97 pgs: 97 active+clean; 218 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-21T04:24:09.586 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:24:09 smithi195 ceph-mon[49435]: pgmap v397: 97 pgs: 97 active+clean; 218 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-21T04:24:11.515 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:24:11 smithi089 ceph-mon[64076]: pgmap v398: 97 pgs: 97 active+clean; 218 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T04:24:11.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:24:11 smithi195 ceph-mon[49435]: pgmap v398: 97 pgs: 97 active+clean; 218 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T04:24:13.515 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:24:13 smithi089 ceph-mon[64076]: pgmap v399: 97 pgs: 97 active+clean; 218 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T04:24:13.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:24:13 smithi195 ceph-mon[49435]: pgmap v399: 97 pgs: 97 active+clean; 218 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T04:24:15.515 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:24:15 smithi089 ceph-mon[64076]: pgmap v400: 97 pgs: 97 active+clean; 218 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T04:24:15.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:24:15 smithi195 ceph-mon[49435]: pgmap v400: 97 pgs: 97 active+clean; 218 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T04:24:16.214 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 1b9b79fc-77ce-11ef-baf5-efdc52797490 -e sha1=f36283e7c91cac1c01cb9a7c4ff21bd697dbd4f2 -- bash -c 'ceph orch ps' 2024-09-21T04:24:17.457 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:24:17 smithi089 ceph-mon[64076]: pgmap v401: 97 pgs: 97 active+clean; 218 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T04:24:17.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:24:17 smithi195 ceph-mon[49435]: pgmap v401: 97 pgs: 97 active+clean; 218 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T04:24:18.194 INFO:teuthology.orchestra.run.smithi089.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-21T04:24:18.194 INFO:teuthology.orchestra.run.smithi089.stdout:alertmanager.smithi089 smithi089 *:9093,9094 running (12m) 97s ago 21m 22.0M - 0.27.0 11f11916f8cd 65dfcbcafee1 2024-09-21T04:24:18.195 INFO:teuthology.orchestra.run.smithi089.stdout:crash.smithi089 smithi089 running (9m) 97s ago 21m 6891k - 19.3.0-5103-gf36283e7 48ed0a2cf89d 6c415a5e173e 2024-09-21T04:24:18.195 INFO:teuthology.orchestra.run.smithi089.stdout:crash.smithi195 smithi195 running (9m) 2m ago 20m 6891k - 19.3.0-5103-gf36283e7 48ed0a2cf89d 78ce05a4e9ab 2024-09-21T04:24:18.195 INFO:teuthology.orchestra.run.smithi089.stdout:grafana.smithi089 smithi089 *:3000 running (105s) 97s ago 21m 43.0M - 10.4.8 5aad1d7cf004 f8eb2943066e 2024-09-21T04:24:18.195 INFO:teuthology.orchestra.run.smithi089.stdout:mds.foofs.smithi089.osufws smithi089 running (2m) 97s ago 15m 33.6M - 19.3.0-5103-gf36283e7 48ed0a2cf89d 526fde771b8f 2024-09-21T04:24:18.195 INFO:teuthology.orchestra.run.smithi089.stdout:mds.foofs.smithi195.hijxre smithi195 running (2m) 2m ago 15m 13.2M - 19.3.0-5103-gf36283e7 48ed0a2cf89d 9b532cb9643e 2024-09-21T04:24:18.195 INFO:teuthology.orchestra.run.smithi089.stdout:mgr.smithi089.srlyxf smithi089 *:8443,9283,8765 running (11m) 97s ago 22m 533M - 19.3.0-5103-gf36283e7 48ed0a2cf89d 3cbeaaa88241 2024-09-21T04:24:18.195 INFO:teuthology.orchestra.run.smithi089.stdout:mgr.smithi195.cgotzm smithi195 *:8443,9283,8765 running (10m) 2m ago 20m 458M - 19.3.0-5103-gf36283e7 48ed0a2cf89d d3734574cf66 2024-09-21T04:24:18.195 INFO:teuthology.orchestra.run.smithi089.stdout:mon.smithi089 smithi089 running (10m) 97s ago 22m 66.3M 2048M 19.3.0-5103-gf36283e7 48ed0a2cf89d 7dd34c8a6f21 2024-09-21T04:24:18.195 INFO:teuthology.orchestra.run.smithi089.stdout:mon.smithi195 smithi195 running (9m) 2m ago 20m 50.2M 2048M 19.3.0-5103-gf36283e7 48ed0a2cf89d 4255937444e7 2024-09-21T04:24:18.195 INFO:teuthology.orchestra.run.smithi089.stdout:nfs.foo.0.0.smithi089.dcjmqc smithi089 *:2049 running (2m) 97s ago 15m 63.1M - 5.9 48ed0a2cf89d 9fd550a02a73 2024-09-21T04:24:18.195 INFO:teuthology.orchestra.run.smithi089.stdout:nfs.foo.1.0.smithi195.vhoayp smithi195 *:2049 running (2m) 2m ago 15m 17.8M - 5.9 48ed0a2cf89d 14ac48ce3073 2024-09-21T04:24:18.195 INFO:teuthology.orchestra.run.smithi089.stdout:node-exporter.smithi089 smithi089 *:9100 running (12m) 97s ago 21m 10.8M - 1.7.0 72c9c2088986 7ef0d1588475 2024-09-21T04:24:18.195 INFO:teuthology.orchestra.run.smithi089.stdout:node-exporter.smithi195 smithi195 *:9100 running (11m) 2m ago 20m 11.1M - 1.7.0 72c9c2088986 61eafd61bc6c 2024-09-21T04:24:18.195 INFO:teuthology.orchestra.run.smithi089.stdout:osd.0 smithi089 running (8m) 97s ago 20m 434M 1935M 19.3.0-5103-gf36283e7 48ed0a2cf89d 3df0adea59fb 2024-09-21T04:24:18.195 INFO:teuthology.orchestra.run.smithi089.stdout:osd.1 smithi089 running (7m) 97s ago 19m 329M 1935M 19.3.0-5103-gf36283e7 48ed0a2cf89d da2e49774e7e 2024-09-21T04:24:18.196 INFO:teuthology.orchestra.run.smithi089.stdout:osd.2 smithi089 running (6m) 97s ago 19m 319M 1935M 19.3.0-5103-gf36283e7 48ed0a2cf89d b9f6438dd5b9 2024-09-21T04:24:18.196 INFO:teuthology.orchestra.run.smithi089.stdout:osd.3 smithi089 running (5m) 97s ago 18m 373M 1935M 19.3.0-5103-gf36283e7 48ed0a2cf89d d269cd94bd08 2024-09-21T04:24:18.196 INFO:teuthology.orchestra.run.smithi089.stdout:osd.4 smithi195 running (5m) 2m ago 18m 467M 2704M 19.3.0-5103-gf36283e7 48ed0a2cf89d e375f90f95c4 2024-09-21T04:24:18.196 INFO:teuthology.orchestra.run.smithi089.stdout:osd.5 smithi195 running (4m) 2m ago 17m 340M 2704M 19.3.0-5103-gf36283e7 48ed0a2cf89d 41de0beb713d 2024-09-21T04:24:18.196 INFO:teuthology.orchestra.run.smithi089.stdout:osd.6 smithi195 running (3m) 2m ago 17m 285M 2704M 19.3.0-5103-gf36283e7 48ed0a2cf89d a41d0c0574ab 2024-09-21T04:24:18.196 INFO:teuthology.orchestra.run.smithi089.stdout:osd.7 smithi195 running (3m) 2m ago 17m 250M 2704M 19.3.0-5103-gf36283e7 48ed0a2cf89d 66d3bf5f6c5f 2024-09-21T04:24:18.196 INFO:teuthology.orchestra.run.smithi089.stdout:prometheus.smithi089 smithi089 *:9095 running (10m) 97s ago 20m 58.8M - 2.51.0 1d3b7f56885b 8c9f309a6459 2024-09-21T04:24:18.515 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:24:18 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T04:24:18.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:24:18 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T04:24:18.823 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 1b9b79fc-77ce-11ef-baf5-efdc52797490 -e sha1=f36283e7c91cac1c01cb9a7c4ff21bd697dbd4f2 -- bash -c 'ceph orch upgrade status' 2024-09-21T04:24:19.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:24:19 smithi195 ceph-mon[49435]: pgmap v402: 97 pgs: 97 active+clean; 218 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-21T04:24:19.359 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:24:19 smithi195 ceph-mon[49435]: from='client.34600 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T04:24:19.378 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:24:19 smithi089 ceph-mon[64076]: pgmap v402: 97 pgs: 97 active+clean; 218 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-21T04:24:19.378 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:24:19 smithi089 ceph-mon[64076]: from='client.34600 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T04:24:20.851 INFO:teuthology.orchestra.run.smithi089.stdout:{ 2024-09-21T04:24:20.852 INFO:teuthology.orchestra.run.smithi089.stdout: "target_image": null, 2024-09-21T04:24:20.852 INFO:teuthology.orchestra.run.smithi089.stdout: "in_progress": false, 2024-09-21T04:24:20.852 INFO:teuthology.orchestra.run.smithi089.stdout: "which": "", 2024-09-21T04:24:20.852 INFO:teuthology.orchestra.run.smithi089.stdout: "services_complete": [], 2024-09-21T04:24:20.852 INFO:teuthology.orchestra.run.smithi089.stdout: "progress": null, 2024-09-21T04:24:20.852 INFO:teuthology.orchestra.run.smithi089.stdout: "message": "", 2024-09-21T04:24:20.852 INFO:teuthology.orchestra.run.smithi089.stdout: "is_paused": false 2024-09-21T04:24:20.852 INFO:teuthology.orchestra.run.smithi089.stdout:} 2024-09-21T04:24:21.515 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:24:21 smithi089 ceph-mon[64076]: pgmap v403: 97 pgs: 97 active+clean; 218 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T04:24:21.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:24:21 smithi195 ceph-mon[49435]: pgmap v403: 97 pgs: 97 active+clean; 218 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T04:24:21.740 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 1b9b79fc-77ce-11ef-baf5-efdc52797490 -e sha1=f36283e7c91cac1c01cb9a7c4ff21bd697dbd4f2 -- bash -c 'ceph health detail' 2024-09-21T04:24:22.290 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:24:22 smithi089 ceph-mon[64076]: from='client.34604 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T04:24:22.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:24:22 smithi195 ceph-mon[49435]: from='client.34604 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T04:24:23.286 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:24:23 smithi089 ceph-mon[64076]: pgmap v404: 97 pgs: 97 active+clean; 218 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T04:24:23.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:24:23 smithi195 ceph-mon[49435]: pgmap v404: 97 pgs: 97 active+clean; 218 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T04:24:23.829 INFO:teuthology.orchestra.run.smithi089.stdout:HEALTH_OK 2024-09-21T04:24:24.432 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:24:24 smithi089 ceph-mon[64076]: from='client.? 172.21.15.89:0/1342166583' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-21T04:24:24.456 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 1b9b79fc-77ce-11ef-baf5-efdc52797490 -e sha1=f36283e7c91cac1c01cb9a7c4ff21bd697dbd4f2 -- bash -c 'ceph versions' 2024-09-21T04:24:24.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:24:24 smithi195 ceph-mon[49435]: from='client.? 172.21.15.89:0/1342166583' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-21T04:24:25.408 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:24:25 smithi089 ceph-mon[64076]: pgmap v405: 97 pgs: 97 active+clean; 218 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T04:24:25.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:24:25 smithi195 ceph-mon[49435]: pgmap v405: 97 pgs: 97 active+clean; 218 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T04:24:26.609 INFO:teuthology.orchestra.run.smithi089.stdout:{ 2024-09-21T04:24:26.610 INFO:teuthology.orchestra.run.smithi089.stdout: "mon": { 2024-09-21T04:24:26.610 INFO:teuthology.orchestra.run.smithi089.stdout: "ceph version 19.3.0-5103-gf36283e7 (f36283e7c91cac1c01cb9a7c4ff21bd697dbd4f2) squid (dev)": 2 2024-09-21T04:24:26.610 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2024-09-21T04:24:26.610 INFO:teuthology.orchestra.run.smithi089.stdout: "mgr": { 2024-09-21T04:24:26.610 INFO:teuthology.orchestra.run.smithi089.stdout: "ceph version 19.3.0-5103-gf36283e7 (f36283e7c91cac1c01cb9a7c4ff21bd697dbd4f2) squid (dev)": 2 2024-09-21T04:24:26.610 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2024-09-21T04:24:26.610 INFO:teuthology.orchestra.run.smithi089.stdout: "osd": { 2024-09-21T04:24:26.610 INFO:teuthology.orchestra.run.smithi089.stdout: "ceph version 19.3.0-5103-gf36283e7 (f36283e7c91cac1c01cb9a7c4ff21bd697dbd4f2) squid (dev)": 8 2024-09-21T04:24:26.610 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2024-09-21T04:24:26.610 INFO:teuthology.orchestra.run.smithi089.stdout: "mds": { 2024-09-21T04:24:26.611 INFO:teuthology.orchestra.run.smithi089.stdout: "ceph version 19.3.0-5103-gf36283e7 (f36283e7c91cac1c01cb9a7c4ff21bd697dbd4f2) squid (dev)": 2 2024-09-21T04:24:26.611 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2024-09-21T04:24:26.611 INFO:teuthology.orchestra.run.smithi089.stdout: "overall": { 2024-09-21T04:24:26.611 INFO:teuthology.orchestra.run.smithi089.stdout: "ceph version 19.3.0-5103-gf36283e7 (f36283e7c91cac1c01cb9a7c4ff21bd697dbd4f2) squid (dev)": 14 2024-09-21T04:24:26.611 INFO:teuthology.orchestra.run.smithi089.stdout: } 2024-09-21T04:24:26.611 INFO:teuthology.orchestra.run.smithi089.stdout:} 2024-09-21T04:24:27.173 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 1b9b79fc-77ce-11ef-baf5-efdc52797490 -e sha1=f36283e7c91cac1c01cb9a7c4ff21bd697dbd4f2 -- bash -c 'ceph versions | jq -e '"'"'.overall | length == 1'"'"'' 2024-09-21T04:24:27.468 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:24:27 smithi089 ceph-mon[64076]: pgmap v406: 97 pgs: 97 active+clean; 218 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T04:24:27.468 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:24:27 smithi089 ceph-mon[64076]: from='client.? 172.21.15.89:0/3362180855' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T04:24:27.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:24:27 smithi195 ceph-mon[49435]: pgmap v406: 97 pgs: 97 active+clean; 218 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T04:24:27.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:24:27 smithi195 ceph-mon[49435]: from='client.? 172.21.15.89:0/3362180855' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T04:24:29.308 INFO:teuthology.orchestra.run.smithi089.stdout:true 2024-09-21T04:24:29.514 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:24:29 smithi089 ceph-mon[64076]: pgmap v407: 97 pgs: 97 active+clean; 218 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-21T04:24:29.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:24:29 smithi195 ceph-mon[49435]: pgmap v407: 97 pgs: 97 active+clean; 218 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-21T04:24:29.914 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 1b9b79fc-77ce-11ef-baf5-efdc52797490 -e sha1=f36283e7c91cac1c01cb9a7c4ff21bd697dbd4f2 -- bash -c 'ceph versions | jq -e '"'"'.overall | keys'"'"' | grep $sha1' 2024-09-21T04:24:30.515 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:24:30 smithi089 ceph-mon[64076]: from='client.? 172.21.15.89:0/107079659' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T04:24:30.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:24:30 smithi195 ceph-mon[49435]: from='client.? 172.21.15.89:0/107079659' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T04:24:31.439 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:24:31 smithi089 ceph-mon[64076]: pgmap v408: 97 pgs: 97 active+clean; 218 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T04:24:31.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:24:31 smithi195 ceph-mon[49435]: pgmap v408: 97 pgs: 97 active+clean; 218 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T04:24:31.982 INFO:teuthology.orchestra.run.smithi089.stdout: "ceph version 19.3.0-5103-gf36283e7 (f36283e7c91cac1c01cb9a7c4ff21bd697dbd4f2) squid (dev)" 2024-09-21T04:24:32.265 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:24:32 smithi089 ceph-mon[64076]: from='client.? 172.21.15.89:0/4270626797' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T04:24:32.585 INFO:teuthology.task.sequential:In sequential, running task cephadm.wait_for_service... 2024-09-21T04:24:32.585 INFO:tasks.cephadm:Waiting for ceph service nfs.foo to start (timeout 300)... 2024-09-21T04:24:32.585 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 1b9b79fc-77ce-11ef-baf5-efdc52797490 -- ceph orch ls -f json 2024-09-21T04:24:32.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:24:32 smithi195 ceph-mon[49435]: from='client.? 172.21.15.89:0/4270626797' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T04:24:33.514 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:24:33 smithi089 ceph-mon[64076]: pgmap v409: 97 pgs: 97 active+clean; 218 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T04:24:33.515 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:24:33 smithi089 ceph-mon[64076]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T04:24:33.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:24:33 smithi195 ceph-mon[49435]: pgmap v409: 97 pgs: 97 active+clean; 218 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T04:24:33.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:24:33 smithi195 ceph-mon[49435]: from='mgr.34104 172.21.15.89:0/1950622799' entity='mgr.smithi089.srlyxf' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T04:24:34.622 INFO:teuthology.orchestra.run.smithi089.stdout: 2024-09-21T04:24:34.622 INFO:teuthology.orchestra.run.smithi089.stdout:[{"placement": {"count": 1}, "service_name": "alertmanager", "service_type": "alertmanager", "status": {"created": "2024-09-21T04:02:13.825693Z", "last_refresh": "2024-09-21T04:22:40.850532Z", "ports": [9093, 9094], "running": 1, "size": 1}}, {"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"created": "2024-09-21T04:02:08.520221Z", "last_refresh": "2024-09-21T04:22:13.614639Z", "running": 2, "size": 2}}, {"placement": {"count": 1}, "service_name": "grafana", "service_type": "grafana", "spec": {"anonymous_access": true, "protocol": "https"}, "status": {"created": "2024-09-21T04:02:10.882761Z", "last_refresh": "2024-09-21T04:22:40.850662Z", "ports": [3000], "running": 1, "size": 1}}, {"placement": {"count": 2}, "service_id": "foofs", "service_name": "mds.foofs", "service_type": "mds", "status": {"created": "2024-09-21T04:08:42.303179Z", "last_refresh": "2024-09-21T04:22:13.615228Z", "running": 2, "size": 2}}, {"placement": {"count": 2}, "service_name": "mgr", "service_type": "mgr", "status": {"created": "2024-09-21T04:02:06.728724Z", "last_refresh": "2024-09-21T04:22:13.614732Z", "running": 2, "size": 2}}, {"placement": {"count": 2, "hosts": ["smithi089:172.21.15.89=smithi089", "smithi195:172.21.15.195=smithi195"]}, "service_name": "mon", "service_type": "mon", "status": {"created": "2024-09-21T04:02:55.539704Z", "last_refresh": "2024-09-21T04:22:13.614805Z", "running": 2, "size": 2}}, {"events": ["2024-09-21T04:23:46.941312Z service:nfs.foo [INFO] \"service was created\""], "placement": {"count": 2}, "service_id": "foo", "service_name": "nfs.foo", "service_type": "nfs", "spec": {"port": 2049}, "status": {"created": "2024-09-21T04:09:04.165173Z", "last_refresh": "2024-09-21T04:22:13.615296Z", "ports": [2049], "running": 2, "size": 2}}, {"placement": {"host_pattern": "*"}, "service_name": "node-exporter", "service_type": "node-exporter", "status": {"created": "2024-09-21T04:02:12.608673Z", "last_refresh": "2024-09-21T04:22:13.614876Z", "ports": [9100], "running": 2, "size": 2}}, {"service_name": "osd", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "48ed0a2cf89d6d0371a2c6185a5869a747265f6abd5ac2a74f00cc32d5628721", "container_image_name": "quay.ceph.io/ceph-ci/ceph@sha256:2195e5f178be0a9390728e2a14088236d8fe6f82635f5c84418fe37a04457e00", "last_refresh": "2024-09-21T04:22:13.614946Z", "running": 8, "size": 8}, "unmanaged": true}, {"placement": {"count": 1}, "service_name": "prometheus", "service_type": "prometheus", "status": {"created": "2024-09-21T04:02:09.740490Z", "last_refresh": "2024-09-21T04:22:40.850790Z", "ports": [9095], "running": 1, "size": 1}}] 2024-09-21T04:24:35.265 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:24:35 smithi089 ceph-mon[64076]: pgmap v410: 97 pgs: 97 active+clean; 218 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T04:24:35.328 INFO:tasks.cephadm:nfs.foo has 2/2 2024-09-21T04:24:35.328 DEBUG:teuthology.parallel:result is None 2024-09-21T04:24:35.328 INFO:teuthology.run_tasks:Running task vip.exec... 2024-09-21T04:24:35.342 INFO:tasks.vip:Running commands on role host.a host ubuntu@smithi089.front.sepia.ceph.com 2024-09-21T04:24:35.342 DEBUG:teuthology.orchestra.run.smithi089:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c 'umount /mnt/foo' 2024-09-21T04:24:35.368 INFO:teuthology.orchestra.run.smithi089.stderr:+ umount /mnt/foo 2024-09-21T04:24:35.396 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-09-21T04:24:35.405 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi089.front.sepia.ceph.com 2024-09-21T04:24:35.405 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 1b9b79fc-77ce-11ef-baf5-efdc52797490 -- bash -c 'ceph nfs cluster ls | grep foo' 2024-09-21T04:24:35.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:24:35 smithi195 ceph-mon[49435]: pgmap v410: 97 pgs: 97 active+clean; 218 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T04:24:36.515 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:24:36 smithi089 ceph-mon[64076]: from='client.34624 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T04:24:36.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:24:36 smithi195 ceph-mon[49435]: from='client.34624 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T04:24:37.510 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:24:37 smithi195 ceph-mon[49435]: pgmap v411: 97 pgs: 97 active+clean; 218 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T04:24:37.515 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:24:37 smithi089 ceph-mon[64076]: pgmap v411: 97 pgs: 97 active+clean; 218 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T04:24:37.651 INFO:teuthology.orchestra.run.smithi089.stdout: "foo" 2024-09-21T04:24:38.335 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 1b9b79fc-77ce-11ef-baf5-efdc52797490 -- bash -c 'ceph nfs export ls foo --detailed' 2024-09-21T04:24:39.515 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:24:39 smithi089 ceph-mon[64076]: from='client.34628 -' entity='client.admin' cmd=[{"prefix": "nfs cluster ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T04:24:39.515 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:24:39 smithi089 ceph-mon[64076]: pgmap v412: 97 pgs: 97 active+clean; 218 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-21T04:24:39.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:24:39 smithi195 ceph-mon[49435]: from='client.34628 -' entity='client.admin' cmd=[{"prefix": "nfs cluster ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T04:24:39.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:24:39 smithi195 ceph-mon[49435]: pgmap v412: 97 pgs: 97 active+clean; 218 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-21T04:24:40.340 INFO:teuthology.orchestra.run.smithi089.stdout:[ 2024-09-21T04:24:40.340 INFO:teuthology.orchestra.run.smithi089.stdout: { 2024-09-21T04:24:40.340 INFO:teuthology.orchestra.run.smithi089.stdout: "access_type": "RW", 2024-09-21T04:24:40.340 INFO:teuthology.orchestra.run.smithi089.stdout: "clients": [], 2024-09-21T04:24:40.340 INFO:teuthology.orchestra.run.smithi089.stdout: "cluster_id": "foo", 2024-09-21T04:24:40.340 INFO:teuthology.orchestra.run.smithi089.stdout: "export_id": 1, 2024-09-21T04:24:40.340 INFO:teuthology.orchestra.run.smithi089.stdout: "fsal": { 2024-09-21T04:24:40.340 INFO:teuthology.orchestra.run.smithi089.stdout: "fs_name": "foofs", 2024-09-21T04:24:40.341 INFO:teuthology.orchestra.run.smithi089.stdout: "name": "CEPH", 2024-09-21T04:24:40.341 INFO:teuthology.orchestra.run.smithi089.stdout: "user_id": "nfs.foo.1" 2024-09-21T04:24:40.341 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2024-09-21T04:24:40.341 INFO:teuthology.orchestra.run.smithi089.stdout: "path": "/", 2024-09-21T04:24:40.341 INFO:teuthology.orchestra.run.smithi089.stdout: "protocols": [ 2024-09-21T04:24:40.341 INFO:teuthology.orchestra.run.smithi089.stdout: 4 2024-09-21T04:24:40.341 INFO:teuthology.orchestra.run.smithi089.stdout: ], 2024-09-21T04:24:40.341 INFO:teuthology.orchestra.run.smithi089.stdout: "pseudo": "/fake", 2024-09-21T04:24:40.341 INFO:teuthology.orchestra.run.smithi089.stdout: "security_label": true, 2024-09-21T04:24:40.341 INFO:teuthology.orchestra.run.smithi089.stdout: "squash": "none", 2024-09-21T04:24:40.341 INFO:teuthology.orchestra.run.smithi089.stdout: "transports": [ 2024-09-21T04:24:40.341 INFO:teuthology.orchestra.run.smithi089.stdout: "TCP" 2024-09-21T04:24:40.341 INFO:teuthology.orchestra.run.smithi089.stdout: ] 2024-09-21T04:24:40.341 INFO:teuthology.orchestra.run.smithi089.stdout: } 2024-09-21T04:24:40.341 INFO:teuthology.orchestra.run.smithi089.stdout:] 2024-09-21T04:24:41.059 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 1b9b79fc-77ce-11ef-baf5-efdc52797490 -- bash -c 'rados -p .nfs --all ls -' 2024-09-21T04:24:41.358 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:24:41 smithi089 ceph-mon[64076]: pgmap v413: 97 pgs: 97 active+clean; 218 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T04:24:41.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:24:41 smithi195 ceph-mon[49435]: pgmap v413: 97 pgs: 97 active+clean; 218 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T04:24:42.358 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:24:42 smithi089 ceph-mon[64076]: from='client.34632 -' entity='client.admin' cmd=[{"prefix": "nfs export ls", "cluster_id": "foo", "detailed": true, "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T04:24:42.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:24:42 smithi195 ceph-mon[49435]: from='client.34632 -' entity='client.admin' cmd=[{"prefix": "nfs export ls", "cluster_id": "foo", "detailed": true, "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T04:24:42.878 INFO:teuthology.orchestra.run.smithi089.stdout:foo rec-0000000000000004:nfs.foo.1 2024-09-21T04:24:42.879 INFO:teuthology.orchestra.run.smithi089.stdout:foo grace 2024-09-21T04:24:42.879 INFO:teuthology.orchestra.run.smithi089.stdout:foo rec-0000000000000004:nfs.foo.0 2024-09-21T04:24:42.879 INFO:teuthology.orchestra.run.smithi089.stdout:foo conf-nfs.foo 2024-09-21T04:24:42.879 INFO:teuthology.orchestra.run.smithi089.stdout:foo export-1 2024-09-21T04:24:43.474 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:24:43 smithi089 ceph-mon[64076]: pgmap v414: 97 pgs: 97 active+clean; 218 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T04:24:43.475 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:24:43 smithi089 ceph-mon[64076]: mgrmap e45: smithi089.srlyxf(active, since 10m), standbys: smithi195.cgotzm 2024-09-21T04:24:43.501 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-09-21T04:24:43.512 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi089.front.sepia.ceph.com 2024-09-21T04:24:43.512 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 1b9b79fc-77ce-11ef-baf5-efdc52797490 -- bash -c 'set -ex 2024-09-21T04:24:43.512 DEBUG:teuthology.orchestra.run.smithi089:> [[ `ceph config get mgr mgr/cephadm/migration_current` -gt 2 ]] 2024-09-21T04:24:43.512 DEBUG:teuthology.orchestra.run.smithi089:> ' 2024-09-21T04:24:43.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:24:43 smithi195 ceph-mon[49435]: pgmap v414: 97 pgs: 97 active+clean; 218 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T04:24:43.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:24:43 smithi195 ceph-mon[49435]: mgrmap e45: smithi089.srlyxf(active, since 10m), standbys: smithi195.cgotzm 2024-09-21T04:24:45.168 INFO:teuthology.orchestra.run.smithi089.stderr:++ ceph config get mgr mgr/cephadm/migration_current 2024-09-21T04:24:45.265 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:24:45 smithi089 ceph-mon[64076]: pgmap v415: 97 pgs: 97 active+clean; 218 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-09-21T04:24:45.439 INFO:teuthology.orchestra.run.smithi089.stderr:+ [[ 7 -gt 2 ]] 2024-09-21T04:24:45.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:24:45 smithi195 ceph-mon[49435]: pgmap v415: 97 pgs: 97 active+clean; 218 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-09-21T04:24:46.063 DEBUG:teuthology.run_tasks:Unwinding manager cephadm 2024-09-21T04:24:46.073 INFO:tasks.cephadm:Teardown begin 2024-09-21T04:24:46.073 DEBUG:teuthology.orchestra.run.smithi089:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-09-21T04:24:46.100 DEBUG:teuthology.orchestra.run.smithi195:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-09-21T04:24:46.134 INFO:tasks.cephadm:Cleaning up testdir ceph.* files... 2024-09-21T04:24:46.134 DEBUG:teuthology.orchestra.run.smithi089:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2024-09-21T04:24:46.156 DEBUG:teuthology.orchestra.run.smithi195:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2024-09-21T04:24:46.191 INFO:tasks.cephadm:Stopping all daemons... 2024-09-21T04:24:46.192 INFO:tasks.cephadm.mon.smithi089:Stopping mon.smithi089... 2024-09-21T04:24:46.192 DEBUG:teuthology.orchestra.run.smithi089:> sudo systemctl stop ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490@mon.smithi089 2024-09-21T04:24:46.494 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:24:46 smithi089 systemd[1]: Stopping Ceph mon.smithi089 for 1b9b79fc-77ce-11ef-baf5-efdc52797490... 2024-09-21T04:24:46.494 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:24:46 smithi089 ceph-mon[64076]: from='client.? 172.21.15.89:0/1831386211' entity='client.admin' cmd={"prefix": "config get", "who": "mgr", "key": "mgr/cephadm/migration_current"} : dispatch 2024-09-21T04:24:46.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:24:46 smithi195 ceph-mon[49435]: from='client.? 172.21.15.89:0/1831386211' entity='client.admin' cmd={"prefix": "config get", "who": "mgr", "key": "mgr/cephadm/migration_current"} : dispatch 2024-09-21T04:24:46.765 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:24:46 smithi089 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-mon-smithi089[64072]: 2024-09-21T04:24:46.490+0000 7fc2fa7db640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.smithi089 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-09-21T04:24:46.765 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:24:46 smithi089 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-mon-smithi089[64072]: 2024-09-21T04:24:46.490+0000 7fc2fa7db640 -1 mon.smithi089@0(leader) e3 *** Got Signal Terminated *** 2024-09-21T04:24:46.765 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:24:46 smithi089 podman[94106]: 2024-09-21 04:24:46.633700802 +0000 UTC m=+0.249244644 container died 7dd34c8a6f21e84b210383e6f843ee23f5d95364b9b5c161c29c2892088e3af1 (image=quay.ceph.io/ceph-ci/ceph@sha256:2195e5f178be0a9390728e2a14088236d8fe6f82635f5c84418fe37a04457e00, name=ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-mon-smithi089, io.buildah.version=1.37.2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=wip-guits-main-2024-09-20-2310-f36283e, ceph=True, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-21T04:24:47.260 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:24:46 smithi089 podman[94106]: 2024-09-21 04:24:46.998889051 +0000 UTC m=+0.614432931 container cleanup 7dd34c8a6f21e84b210383e6f843ee23f5d95364b9b5c161c29c2892088e3af1 (image=quay.ceph.io/ceph-ci/ceph@sha256:2195e5f178be0a9390728e2a14088236d8fe6f82635f5c84418fe37a04457e00, name=ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-mon-smithi089, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, RELEASE=wip-guits-main-2024-09-20-2310-f36283e, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD) 2024-09-21T04:24:47.260 INFO:journalctl@ceph.mon.smithi089.smithi089.stdout:Sep 21 04:24:46 smithi089 bash[94106]: ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-mon-smithi089 2024-09-21T04:24:47.438 DEBUG:teuthology.orchestra.run.smithi089:> sudo pkill -f 'journalctl -f -n 0 -u ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490@mon.smithi089.service' 2024-09-21T04:24:47.492 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-21T04:24:47.492 INFO:tasks.cephadm.mon.smithi089:Stopped mon.smithi089 2024-09-21T04:24:47.493 INFO:tasks.cephadm.mon.smithi195:Stopping mon.smithi195... 2024-09-21T04:24:47.493 DEBUG:teuthology.orchestra.run.smithi195:> sudo systemctl stop ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490@mon.smithi195 2024-09-21T04:24:47.784 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:24:47 smithi195 systemd[1]: Stopping Ceph mon.smithi195 for 1b9b79fc-77ce-11ef-baf5-efdc52797490... 2024-09-21T04:24:48.109 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:24:47 smithi195 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-mon-smithi195[49431]: 2024-09-21T04:24:47.780+0000 7f9391614640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.smithi195 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-09-21T04:24:48.109 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:24:47 smithi195 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-mon-smithi195[49431]: 2024-09-21T04:24:47.780+0000 7f9391614640 -1 mon.smithi195@1(peon) e3 *** Got Signal Terminated *** 2024-09-21T04:24:48.109 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:24:47 smithi195 podman[68904]: 2024-09-21 04:24:47.91444538 +0000 UTC m=+0.232098160 container died 4255937444e7daceb33d28074f518a4cf14f676d8f19b06c8d560d11769055c3 (image=quay.ceph.io/ceph-ci/ceph@sha256:2195e5f178be0a9390728e2a14088236d8fe6f82635f5c84418fe37a04457e00, name=ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-mon-smithi195, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, RELEASE=wip-guits-main-2024-09-20-2310-f36283e, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.license=GPLv2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-21T04:24:48.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:24:48 smithi195 podman[68904]: 2024-09-21 04:24:48.362565524 +0000 UTC m=+0.680218301 container cleanup 4255937444e7daceb33d28074f518a4cf14f676d8f19b06c8d560d11769055c3 (image=quay.ceph.io/ceph-ci/ceph@sha256:2195e5f178be0a9390728e2a14088236d8fe6f82635f5c84418fe37a04457e00, name=ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-mon-smithi195, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, ceph=True, RELEASE=wip-guits-main-2024-09-20-2310-f36283e, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True) 2024-09-21T04:24:48.609 INFO:journalctl@ceph.mon.smithi195.smithi195.stdout:Sep 21 04:24:48 smithi195 bash[68904]: ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-mon-smithi195 2024-09-21T04:24:48.790 DEBUG:teuthology.orchestra.run.smithi195:> sudo pkill -f 'journalctl -f -n 0 -u ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490@mon.smithi195.service' 2024-09-21T04:24:48.854 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-21T04:24:48.855 INFO:tasks.cephadm.mon.smithi195:Stopped mon.smithi195 2024-09-21T04:24:48.855 INFO:tasks.cephadm.osd.0:Stopping osd.0... 2024-09-21T04:24:48.855 DEBUG:teuthology.orchestra.run.smithi089:> sudo systemctl stop ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490@osd.0 2024-09-21T04:24:49.148 INFO:journalctl@ceph.osd.0.smithi089.stdout:Sep 21 04:24:48 smithi089 systemd[1]: Stopping Ceph osd.0 for 1b9b79fc-77ce-11ef-baf5-efdc52797490... 2024-09-21T04:24:49.515 INFO:journalctl@ceph.osd.0.smithi089.stdout:Sep 21 04:24:49 smithi089 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-0[68909]: 2024-09-21T04:24:49.144+0000 7f4f0f296640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-21T04:24:49.515 INFO:journalctl@ceph.osd.0.smithi089.stdout:Sep 21 04:24:49 smithi089 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-0[68909]: 2024-09-21T04:24:49.144+0000 7f4f0f296640 -1 osd.0 165 *** Got signal Terminated *** 2024-09-21T04:24:49.515 INFO:journalctl@ceph.osd.0.smithi089.stdout:Sep 21 04:24:49 smithi089 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-0[68909]: 2024-09-21T04:24:49.144+0000 7f4f0f296640 -1 osd.0 165 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-21T04:24:54.900 INFO:journalctl@ceph.osd.0.smithi089.stdout:Sep 21 04:24:54 smithi089 podman[94237]: 2024-09-21 04:24:54.625801334 +0000 UTC m=+5.580850594 container died 3df0adea59fbe887dc16f6fe4727ab17845fbb1a2e133b9e8cbeae51f91f0160 (image=quay.ceph.io/ceph-ci/ceph@sha256:2195e5f178be0a9390728e2a14088236d8fe6f82635f5c84418fe37a04457e00, name=ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-0, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, ceph=True, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, RELEASE=wip-guits-main-2024-09-20-2310-f36283e, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2) 2024-09-21T04:24:55.240 INFO:journalctl@ceph.osd.0.smithi089.stdout:Sep 21 04:24:54 smithi089 podman[94237]: 2024-09-21 04:24:54.979054084 +0000 UTC m=+5.934103347 container cleanup 3df0adea59fbe887dc16f6fe4727ab17845fbb1a2e133b9e8cbeae51f91f0160 (image=quay.ceph.io/ceph-ci/ceph@sha256:2195e5f178be0a9390728e2a14088236d8fe6f82635f5c84418fe37a04457e00, name=ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-0, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, RELEASE=wip-guits-main-2024-09-20-2310-f36283e, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, io.buildah.version=1.37.2) 2024-09-21T04:24:55.241 INFO:journalctl@ceph.osd.0.smithi089.stdout:Sep 21 04:24:54 smithi089 bash[94237]: ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-0 2024-09-21T04:24:55.515 INFO:journalctl@ceph.osd.0.smithi089.stdout:Sep 21 04:24:55 smithi089 podman[94267]: 2024-09-21 04:24:55.238672803 +0000 UTC m=+0.612768901 container remove 3df0adea59fbe887dc16f6fe4727ab17845fbb1a2e133b9e8cbeae51f91f0160 (image=quay.ceph.io/ceph-ci/ceph@sha256:2195e5f178be0a9390728e2a14088236d8fe6f82635f5c84418fe37a04457e00, name=ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-0, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, RELEASE=wip-guits-main-2024-09-20-2310-f36283e, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, GIT_CLEAN=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23) 2024-09-21T04:24:55.895 INFO:journalctl@ceph.osd.0.smithi089.stdout:Sep 21 04:24:55 smithi089 podman[94359]: 2024-09-21 04:24:55.493469877 +0000 UTC m=+0.016308177 image pull 48ed0a2cf89d6d0371a2c6185a5869a747265f6abd5ac2a74f00cc32d5628721 quay.ceph.io/ceph-ci/ceph@sha256:2195e5f178be0a9390728e2a14088236d8fe6f82635f5c84418fe37a04457e00 2024-09-21T04:24:55.895 INFO:journalctl@ceph.osd.0.smithi089.stdout:Sep 21 04:24:55 smithi089 podman[94359]: 2024-09-21 04:24:55.606277285 +0000 UTC m=+0.129115575 container create 0fc033d0aea16a7acae22599c2629c811dd386c415d8d9998700ae49188ad20e (image=quay.ceph.io/ceph-ci/ceph@sha256:2195e5f178be0a9390728e2a14088236d8fe6f82635f5c84418fe37a04457e00, name=ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-0-deactivate, io.buildah.version=1.37.2, ceph=True, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, RELEASE=wip-guits-main-2024-09-20-2310-f36283e, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_CLEAN=True) 2024-09-21T04:24:55.895 INFO:journalctl@ceph.osd.0.smithi089.stdout:Sep 21 04:24:55 smithi089 podman[94359]: 2024-09-21 04:24:55.808851357 +0000 UTC m=+0.331689651 container init 0fc033d0aea16a7acae22599c2629c811dd386c415d8d9998700ae49188ad20e (image=quay.ceph.io/ceph-ci/ceph@sha256:2195e5f178be0a9390728e2a14088236d8fe6f82635f5c84418fe37a04457e00, name=ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-0-deactivate, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_CLEAN=True, org.label-schema.license=GPLv2, ceph=True, org.label-schema.vendor=CentOS, RELEASE=wip-guits-main-2024-09-20-2310-f36283e, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux ) 2024-09-21T04:24:55.896 INFO:journalctl@ceph.osd.0.smithi089.stdout:Sep 21 04:24:55 smithi089 podman[94359]: 2024-09-21 04:24:55.812146561 +0000 UTC m=+0.334984861 container start 0fc033d0aea16a7acae22599c2629c811dd386c415d8d9998700ae49188ad20e (image=quay.ceph.io/ceph-ci/ceph@sha256:2195e5f178be0a9390728e2a14088236d8fe6f82635f5c84418fe37a04457e00, name=ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-0-deactivate, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.build-date=20240731, RELEASE=wip-guits-main-2024-09-20-2310-f36283e, org.label-schema.license=GPLv2, ceph=True, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD) 2024-09-21T04:24:56.265 INFO:journalctl@ceph.osd.0.smithi089.stdout:Sep 21 04:24:55 smithi089 podman[94359]: 2024-09-21 04:24:55.892355093 +0000 UTC m=+0.415193401 container attach 0fc033d0aea16a7acae22599c2629c811dd386c415d8d9998700ae49188ad20e (image=quay.ceph.io/ceph-ci/ceph@sha256:2195e5f178be0a9390728e2a14088236d8fe6f82635f5c84418fe37a04457e00, name=ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-0-deactivate, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, GIT_CLEAN=True, RELEASE=wip-guits-main-2024-09-20-2310-f36283e, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True) 2024-09-21T04:24:56.265 INFO:journalctl@ceph.osd.0.smithi089.stdout:Sep 21 04:24:56 smithi089 podman[94359]: 2024-09-21 04:24:56.001003727 +0000 UTC m=+0.523842029 container died 0fc033d0aea16a7acae22599c2629c811dd386c415d8d9998700ae49188ad20e (image=quay.ceph.io/ceph-ci/ceph@sha256:2195e5f178be0a9390728e2a14088236d8fe6f82635f5c84418fe37a04457e00, name=ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-0-deactivate, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , RELEASE=wip-guits-main-2024-09-20-2310-f36283e, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, ceph=True, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD) 2024-09-21T04:24:56.634 INFO:journalctl@ceph.osd.0.smithi089.stdout:Sep 21 04:24:56 smithi089 podman[94383]: 2024-09-21 04:24:56.613355325 +0000 UTC m=+0.606560048 container remove 0fc033d0aea16a7acae22599c2629c811dd386c415d8d9998700ae49188ad20e (image=quay.ceph.io/ceph-ci/ceph@sha256:2195e5f178be0a9390728e2a14088236d8fe6f82635f5c84418fe37a04457e00, name=ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-0-deactivate, RELEASE=wip-guits-main-2024-09-20-2310-f36283e, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, ceph=True) 2024-09-21T04:24:56.639 DEBUG:teuthology.orchestra.run.smithi089:> sudo pkill -f 'journalctl -f -n 0 -u ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490@osd.0.service' 2024-09-21T04:24:56.680 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-21T04:24:56.680 INFO:tasks.cephadm.osd.0:Stopped osd.0 2024-09-21T04:24:56.681 INFO:tasks.cephadm.osd.1:Stopping osd.1... 2024-09-21T04:24:56.681 DEBUG:teuthology.orchestra.run.smithi089:> sudo systemctl stop ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490@osd.1 2024-09-21T04:24:57.015 INFO:journalctl@ceph.osd.1.smithi089.stdout:Sep 21 04:24:56 smithi089 systemd[1]: Stopping Ceph osd.1 for 1b9b79fc-77ce-11ef-baf5-efdc52797490... 2024-09-21T04:24:57.515 INFO:journalctl@ceph.osd.1.smithi089.stdout:Sep 21 04:24:57 smithi089 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-1[71474]: 2024-09-21T04:24:57.108+0000 7f4c9f751640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.1 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-21T04:24:57.515 INFO:journalctl@ceph.osd.1.smithi089.stdout:Sep 21 04:24:57 smithi089 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-1[71474]: 2024-09-21T04:24:57.108+0000 7f4c9f751640 -1 osd.1 165 *** Got signal Terminated *** 2024-09-21T04:24:57.515 INFO:journalctl@ceph.osd.1.smithi089.stdout:Sep 21 04:24:57 smithi089 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-1[71474]: 2024-09-21T04:24:57.108+0000 7f4c9f751640 -1 osd.1 165 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-21T04:25:02.901 INFO:journalctl@ceph.osd.1.smithi089.stdout:Sep 21 04:25:02 smithi089 podman[94464]: 2024-09-21 04:25:02.600799638 +0000 UTC m=+5.704944941 container died da2e49774e7ec1d3618c3ed46871ffb1cd2bf3b990986548c2d891e2c3054e66 (image=quay.ceph.io/ceph-ci/ceph@sha256:2195e5f178be0a9390728e2a14088236d8fe6f82635f5c84418fe37a04457e00, name=ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-1, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.build-date=20240731, RELEASE=wip-guits-main-2024-09-20-2310-f36283e, ceph=True, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0) 2024-09-21T04:25:03.178 INFO:journalctl@ceph.osd.1.smithi089.stdout:Sep 21 04:25:02 smithi089 podman[94464]: 2024-09-21 04:25:02.899241269 +0000 UTC m=+6.003386573 container cleanup da2e49774e7ec1d3618c3ed46871ffb1cd2bf3b990986548c2d891e2c3054e66 (image=quay.ceph.io/ceph-ci/ceph@sha256:2195e5f178be0a9390728e2a14088236d8fe6f82635f5c84418fe37a04457e00, name=ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-1, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, RELEASE=wip-guits-main-2024-09-20-2310-f36283e, ceph=True, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_CLEAN=True, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0) 2024-09-21T04:25:03.178 INFO:journalctl@ceph.osd.1.smithi089.stdout:Sep 21 04:25:02 smithi089 bash[94464]: ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-1 2024-09-21T04:25:03.515 INFO:journalctl@ceph.osd.1.smithi089.stdout:Sep 21 04:25:03 smithi089 podman[94479]: 2024-09-21 04:25:03.176045999 +0000 UTC m=+0.568713874 container remove da2e49774e7ec1d3618c3ed46871ffb1cd2bf3b990986548c2d891e2c3054e66 (image=quay.ceph.io/ceph-ci/ceph@sha256:2195e5f178be0a9390728e2a14088236d8fe6f82635f5c84418fe37a04457e00, name=ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-1, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, RELEASE=wip-guits-main-2024-09-20-2310-f36283e, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, org.label-schema.schema-version=1.0) 2024-09-21T04:25:03.898 INFO:journalctl@ceph.osd.1.smithi089.stdout:Sep 21 04:25:03 smithi089 podman[94571]: 2024-09-21 04:25:03.435566229 +0000 UTC m=+0.016252909 image pull 48ed0a2cf89d6d0371a2c6185a5869a747265f6abd5ac2a74f00cc32d5628721 quay.ceph.io/ceph-ci/ceph@sha256:2195e5f178be0a9390728e2a14088236d8fe6f82635f5c84418fe37a04457e00 2024-09-21T04:25:03.899 INFO:journalctl@ceph.osd.1.smithi089.stdout:Sep 21 04:25:03 smithi089 podman[94571]: 2024-09-21 04:25:03.560264329 +0000 UTC m=+0.140951010 container create a026b992aa197a33e571311aab4aead11377a30533aca161db36175e7a5ea79c (image=quay.ceph.io/ceph-ci/ceph@sha256:2195e5f178be0a9390728e2a14088236d8fe6f82635f5c84418fe37a04457e00, name=ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-1-deactivate, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-guits-main-2024-09-20-2310-f36283e, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , ceph=True) 2024-09-21T04:25:03.899 INFO:journalctl@ceph.osd.1.smithi089.stdout:Sep 21 04:25:03 smithi089 podman[94571]: 2024-09-21 04:25:03.771181399 +0000 UTC m=+0.351868091 container init a026b992aa197a33e571311aab4aead11377a30533aca161db36175e7a5ea79c (image=quay.ceph.io/ceph-ci/ceph@sha256:2195e5f178be0a9390728e2a14088236d8fe6f82635f5c84418fe37a04457e00, name=ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-1-deactivate, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=wip-guits-main-2024-09-20-2310-f36283e, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_CLEAN=True, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-21T04:25:03.899 INFO:journalctl@ceph.osd.1.smithi089.stdout:Sep 21 04:25:03 smithi089 podman[94571]: 2024-09-21 04:25:03.77476487 +0000 UTC m=+0.355451552 container start a026b992aa197a33e571311aab4aead11377a30533aca161db36175e7a5ea79c (image=quay.ceph.io/ceph-ci/ceph@sha256:2195e5f178be0a9390728e2a14088236d8fe6f82635f5c84418fe37a04457e00, name=ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-1-deactivate, ceph=True, RELEASE=wip-guits-main-2024-09-20-2310-f36283e, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, maintainer=Guillaume Abrioux ) 2024-09-21T04:25:03.899 INFO:journalctl@ceph.osd.1.smithi089.stdout:Sep 21 04:25:03 smithi089 podman[94571]: 2024-09-21 04:25:03.896002505 +0000 UTC m=+0.476689189 container attach a026b992aa197a33e571311aab4aead11377a30533aca161db36175e7a5ea79c (image=quay.ceph.io/ceph-ci/ceph@sha256:2195e5f178be0a9390728e2a14088236d8fe6f82635f5c84418fe37a04457e00, name=ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-1-deactivate, org.label-schema.schema-version=1.0, RELEASE=wip-guits-main-2024-09-20-2310-f36283e, maintainer=Guillaume Abrioux , GIT_CLEAN=True, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-21T04:25:04.251 INFO:journalctl@ceph.osd.1.smithi089.stdout:Sep 21 04:25:03 smithi089 podman[94571]: 2024-09-21 04:25:03.968347803 +0000 UTC m=+0.549034485 container died a026b992aa197a33e571311aab4aead11377a30533aca161db36175e7a5ea79c (image=quay.ceph.io/ceph-ci/ceph@sha256:2195e5f178be0a9390728e2a14088236d8fe6f82635f5c84418fe37a04457e00, name=ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-1-deactivate, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, ceph=True, RELEASE=wip-guits-main-2024-09-20-2310-f36283e, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS) 2024-09-21T04:25:04.502 INFO:journalctl@ceph.osd.1.smithi089.stdout:Sep 21 04:25:04 smithi089 podman[94595]: 2024-09-21 04:25:04.483286585 +0000 UTC m=+0.508985914 container remove a026b992aa197a33e571311aab4aead11377a30533aca161db36175e7a5ea79c (image=quay.ceph.io/ceph-ci/ceph@sha256:2195e5f178be0a9390728e2a14088236d8fe6f82635f5c84418fe37a04457e00, name=ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-1-deactivate, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, GIT_CLEAN=True, io.buildah.version=1.37.2, RELEASE=wip-guits-main-2024-09-20-2310-f36283e, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2) 2024-09-21T04:25:04.507 DEBUG:teuthology.orchestra.run.smithi089:> sudo pkill -f 'journalctl -f -n 0 -u ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490@osd.1.service' 2024-09-21T04:25:04.545 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-21T04:25:04.545 INFO:tasks.cephadm.osd.1:Stopped osd.1 2024-09-21T04:25:04.545 INFO:tasks.cephadm.osd.2:Stopping osd.2... 2024-09-21T04:25:04.546 DEBUG:teuthology.orchestra.run.smithi089:> sudo systemctl stop ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490@osd.2 2024-09-21T04:25:04.765 INFO:journalctl@ceph.osd.2.smithi089.stdout:Sep 21 04:25:04 smithi089 systemd[1]: Stopping Ceph osd.2 for 1b9b79fc-77ce-11ef-baf5-efdc52797490... 2024-09-21T04:25:05.265 INFO:journalctl@ceph.osd.2.smithi089.stdout:Sep 21 04:25:04 smithi089 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-2[73905]: 2024-09-21T04:25:04.869+0000 7f4f1bdea640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.2 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-21T04:25:05.265 INFO:journalctl@ceph.osd.2.smithi089.stdout:Sep 21 04:25:04 smithi089 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-2[73905]: 2024-09-21T04:25:04.869+0000 7f4f1bdea640 -1 osd.2 165 *** Got signal Terminated *** 2024-09-21T04:25:05.265 INFO:journalctl@ceph.osd.2.smithi089.stdout:Sep 21 04:25:04 smithi089 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-2[73905]: 2024-09-21T04:25:04.869+0000 7f4f1bdea640 -1 osd.2 165 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-21T04:25:10.515 INFO:journalctl@ceph.osd.2.smithi089.stdout:Sep 21 04:25:10 smithi089 podman[94676]: 2024-09-21 04:25:10.31795061 +0000 UTC m=+5.547541649 container died b9f6438dd5b9af6ba60900e659361b714e9c8e5631963f8c00cdabc7516099a5 (image=quay.ceph.io/ceph-ci/ceph@sha256:2195e5f178be0a9390728e2a14088236d8fe6f82635f5c84418fe37a04457e00, name=ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-2, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=wip-guits-main-2024-09-20-2310-f36283e) 2024-09-21T04:25:10.956 INFO:journalctl@ceph.osd.2.smithi089.stdout:Sep 21 04:25:10 smithi089 podman[94676]: 2024-09-21 04:25:10.694587198 +0000 UTC m=+5.924178236 container cleanup b9f6438dd5b9af6ba60900e659361b714e9c8e5631963f8c00cdabc7516099a5 (image=quay.ceph.io/ceph-ci/ceph@sha256:2195e5f178be0a9390728e2a14088236d8fe6f82635f5c84418fe37a04457e00, name=ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-2, RELEASE=wip-guits-main-2024-09-20-2310-f36283e, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_CLEAN=True, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD) 2024-09-21T04:25:10.956 INFO:journalctl@ceph.osd.2.smithi089.stdout:Sep 21 04:25:10 smithi089 bash[94676]: ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-2 2024-09-21T04:25:11.265 INFO:journalctl@ceph.osd.2.smithi089.stdout:Sep 21 04:25:10 smithi089 podman[94692]: 2024-09-21 04:25:10.954291149 +0000 UTC m=+0.638515007 container remove b9f6438dd5b9af6ba60900e659361b714e9c8e5631963f8c00cdabc7516099a5 (image=quay.ceph.io/ceph-ci/ceph@sha256:2195e5f178be0a9390728e2a14088236d8fe6f82635f5c84418fe37a04457e00, name=ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-2, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, ceph=True, GIT_CLEAN=True, org.label-schema.build-date=20240731, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, RELEASE=wip-guits-main-2024-09-20-2310-f36283e, maintainer=Guillaume Abrioux ) 2024-09-21T04:25:11.569 INFO:journalctl@ceph.osd.2.smithi089.stdout:Sep 21 04:25:11 smithi089 podman[94779]: 2024-09-21 04:25:11.263497053 +0000 UTC m=+0.110672252 container create b633d02cf808ebeca7cd9d85fdc1de1755e41f521f175269ccd2ffdbfaa9f692 (image=quay.ceph.io/ceph-ci/ceph@sha256:2195e5f178be0a9390728e2a14088236d8fe6f82635f5c84418fe37a04457e00, name=ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-2-deactivate, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, RELEASE=wip-guits-main-2024-09-20-2310-f36283e, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-21T04:25:11.569 INFO:journalctl@ceph.osd.2.smithi089.stdout:Sep 21 04:25:11 smithi089 podman[94779]: 2024-09-21 04:25:11.16919297 +0000 UTC m=+0.016368185 image pull 48ed0a2cf89d6d0371a2c6185a5869a747265f6abd5ac2a74f00cc32d5628721 quay.ceph.io/ceph-ci/ceph@sha256:2195e5f178be0a9390728e2a14088236d8fe6f82635f5c84418fe37a04457e00 2024-09-21T04:25:11.569 INFO:journalctl@ceph.osd.2.smithi089.stdout:Sep 21 04:25:11 smithi089 podman[94779]: 2024-09-21 04:25:11.483171442 +0000 UTC m=+0.330346651 container init b633d02cf808ebeca7cd9d85fdc1de1755e41f521f175269ccd2ffdbfaa9f692 (image=quay.ceph.io/ceph-ci/ceph@sha256:2195e5f178be0a9390728e2a14088236d8fe6f82635f5c84418fe37a04457e00, name=ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-2-deactivate, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, RELEASE=wip-guits-main-2024-09-20-2310-f36283e, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.license=GPLv2) 2024-09-21T04:25:11.569 INFO:journalctl@ceph.osd.2.smithi089.stdout:Sep 21 04:25:11 smithi089 podman[94779]: 2024-09-21 04:25:11.486825042 +0000 UTC m=+0.334000252 container start b633d02cf808ebeca7cd9d85fdc1de1755e41f521f175269ccd2ffdbfaa9f692 (image=quay.ceph.io/ceph-ci/ceph@sha256:2195e5f178be0a9390728e2a14088236d8fe6f82635f5c84418fe37a04457e00, name=ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-2-deactivate, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, ceph=True, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, RELEASE=wip-guits-main-2024-09-20-2310-f36283e, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0) 2024-09-21T04:25:11.569 INFO:journalctl@ceph.osd.2.smithi089.stdout:Sep 21 04:25:11 smithi089 podman[94779]: 2024-09-21 04:25:11.566135357 +0000 UTC m=+0.413310585 container attach b633d02cf808ebeca7cd9d85fdc1de1755e41f521f175269ccd2ffdbfaa9f692 (image=quay.ceph.io/ceph-ci/ceph@sha256:2195e5f178be0a9390728e2a14088236d8fe6f82635f5c84418fe37a04457e00, name=ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-2-deactivate, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-guits-main-2024-09-20-2310-f36283e, org.label-schema.build-date=20240731, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, ceph=True) 2024-09-21T04:25:11.964 INFO:journalctl@ceph.osd.2.smithi089.stdout:Sep 21 04:25:11 smithi089 podman[94779]: 2024-09-21 04:25:11.671278244 +0000 UTC m=+0.518453449 container died b633d02cf808ebeca7cd9d85fdc1de1755e41f521f175269ccd2ffdbfaa9f692 (image=quay.ceph.io/ceph-ci/ceph@sha256:2195e5f178be0a9390728e2a14088236d8fe6f82635f5c84418fe37a04457e00, name=ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-2-deactivate, ceph=True, org.label-schema.license=GPLv2, RELEASE=wip-guits-main-2024-09-20-2310-f36283e, maintainer=Guillaume Abrioux , GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-21T04:25:12.269 DEBUG:teuthology.orchestra.run.smithi089:> sudo pkill -f 'journalctl -f -n 0 -u ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490@osd.2.service' 2024-09-21T04:25:12.307 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-21T04:25:12.307 INFO:tasks.cephadm.osd.2:Stopped osd.2 2024-09-21T04:25:12.307 INFO:tasks.cephadm.osd.3:Stopping osd.3... 2024-09-21T04:25:12.307 DEBUG:teuthology.orchestra.run.smithi089:> sudo systemctl stop ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490@osd.3 2024-09-21T04:25:12.515 INFO:journalctl@ceph.osd.3.smithi089.stdout:Sep 21 04:25:12 smithi089 systemd[1]: Stopping Ceph osd.3 for 1b9b79fc-77ce-11ef-baf5-efdc52797490... 2024-09-21T04:25:13.015 INFO:journalctl@ceph.osd.3.smithi089.stdout:Sep 21 04:25:12 smithi089 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-3[76462]: 2024-09-21T04:25:12.630+0000 7f2dcff9f640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.3 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-21T04:25:13.015 INFO:journalctl@ceph.osd.3.smithi089.stdout:Sep 21 04:25:12 smithi089 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-3[76462]: 2024-09-21T04:25:12.630+0000 7f2dcff9f640 -1 osd.3 165 *** Got signal Terminated *** 2024-09-21T04:25:13.015 INFO:journalctl@ceph.osd.3.smithi089.stdout:Sep 21 04:25:12 smithi089 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-3[76462]: 2024-09-21T04:25:12.630+0000 7f2dcff9f640 -1 osd.3 165 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-21T04:25:16.765 INFO:journalctl@ceph.osd.3.smithi089.stdout:Sep 21 04:25:16 smithi089 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-3[76462]: 2024-09-21T04:25:16.418+0000 7f2dcc5b4640 -1 osd.3 165 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2024-09-21T04:24:52.681597+0000 front 2024-09-21T04:24:52.681493+0000 (oldest deadline 2024-09-21T04:25:15.581614+0000) 2024-09-21T04:25:17.765 INFO:journalctl@ceph.osd.3.smithi089.stdout:Sep 21 04:25:17 smithi089 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-3[76462]: 2024-09-21T04:25:17.395+0000 7f2dcc5b4640 -1 osd.3 165 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2024-09-21T04:24:52.681597+0000 front 2024-09-21T04:24:52.681493+0000 (oldest deadline 2024-09-21T04:25:15.581614+0000) 2024-09-21T04:25:18.394 INFO:journalctl@ceph.osd.3.smithi089.stdout:Sep 21 04:25:18 smithi089 podman[94882]: 2024-09-21 04:25:18.102901126 +0000 UTC m=+5.573048110 container died d269cd94bd08f232fd3d269a8e6469ecf78ba2c0c91cefff44da5a2f66bdd9c0 (image=quay.ceph.io/ceph-ci/ceph@sha256:2195e5f178be0a9390728e2a14088236d8fe6f82635f5c84418fe37a04457e00, name=ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-3, RELEASE=wip-guits-main-2024-09-20-2310-f36283e, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , GIT_CLEAN=True, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, CEPH_POINT_RELEASE=) 2024-09-21T04:25:18.726 INFO:journalctl@ceph.osd.3.smithi089.stdout:Sep 21 04:25:18 smithi089 podman[94882]: 2024-09-21 04:25:18.472588641 +0000 UTC m=+5.942735614 container cleanup d269cd94bd08f232fd3d269a8e6469ecf78ba2c0c91cefff44da5a2f66bdd9c0 (image=quay.ceph.io/ceph-ci/ceph@sha256:2195e5f178be0a9390728e2a14088236d8fe6f82635f5c84418fe37a04457e00, name=ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-3, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.build-date=20240731, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, RELEASE=wip-guits-main-2024-09-20-2310-f36283e) 2024-09-21T04:25:18.726 INFO:journalctl@ceph.osd.3.smithi089.stdout:Sep 21 04:25:18 smithi089 bash[94882]: ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-3 2024-09-21T04:25:18.789 INFO:journalctl@ceph.osd.4.smithi195.stdout:Sep 21 04:25:18 smithi195 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-4[53378]: 2024-09-21T04:25:18.475+0000 7fda08ec7640 -1 osd.4 165 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2024-09-21T04:24:52.076328+0000 front 2024-09-21T04:24:52.076253+0000 (oldest deadline 2024-09-21T04:25:17.976096+0000) 2024-09-21T04:25:18.790 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 21 04:25:18 smithi195 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-7[59664]: 2024-09-21T04:25:18.675+0000 7f99a543c640 -1 osd.7 165 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2024-09-21T04:24:52.054834+0000 front 2024-09-21T04:24:52.054715+0000 (oldest deadline 2024-09-21T04:25:17.855130+0000) 2024-09-21T04:25:19.015 INFO:journalctl@ceph.osd.3.smithi089.stdout:Sep 21 04:25:18 smithi089 podman[94914]: 2024-09-21 04:25:18.724072194 +0000 UTC m=+0.615478647 container remove d269cd94bd08f232fd3d269a8e6469ecf78ba2c0c91cefff44da5a2f66bdd9c0 (image=quay.ceph.io/ceph-ci/ceph@sha256:2195e5f178be0a9390728e2a14088236d8fe6f82635f5c84418fe37a04457e00, name=ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-3, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, ceph=True, CEPH_POINT_RELEASE=, RELEASE=wip-guits-main-2024-09-20-2310-f36283e, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux ) 2024-09-21T04:25:19.109 INFO:journalctl@ceph.osd.6.smithi195.stdout:Sep 21 04:25:18 smithi195 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-6[57554]: 2024-09-21T04:25:18.786+0000 7f9870ab3640 -1 osd.6 165 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2024-09-21T04:24:52.395503+0000 front 2024-09-21T04:24:52.395593+0000 (oldest deadline 2024-09-21T04:25:18.295679+0000) 2024-09-21T04:25:19.380 INFO:journalctl@ceph.osd.3.smithi089.stdout:Sep 21 04:25:19 smithi089 podman[95003]: 2024-09-21 04:25:19.075076089 +0000 UTC m=+0.111980117 container create 7b65b7c2fdca480f6c719e8f48907e22ba8bc740cd4af503c2686292703a0dee (image=quay.ceph.io/ceph-ci/ceph@sha256:2195e5f178be0a9390728e2a14088236d8fe6f82635f5c84418fe37a04457e00, name=ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-3-deactivate, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, ceph=True, RELEASE=wip-guits-main-2024-09-20-2310-f36283e, GIT_CLEAN=True) 2024-09-21T04:25:19.380 INFO:journalctl@ceph.osd.3.smithi089.stdout:Sep 21 04:25:19 smithi089 podman[95003]: 2024-09-21 04:25:18.979274397 +0000 UTC m=+0.016178443 image pull 48ed0a2cf89d6d0371a2c6185a5869a747265f6abd5ac2a74f00cc32d5628721 quay.ceph.io/ceph-ci/ceph@sha256:2195e5f178be0a9390728e2a14088236d8fe6f82635f5c84418fe37a04457e00 2024-09-21T04:25:19.380 INFO:journalctl@ceph.osd.3.smithi089.stdout:Sep 21 04:25:19 smithi089 podman[95003]: 2024-09-21 04:25:19.294814407 +0000 UTC m=+0.331718436 container init 7b65b7c2fdca480f6c719e8f48907e22ba8bc740cd4af503c2686292703a0dee (image=quay.ceph.io/ceph-ci/ceph@sha256:2195e5f178be0a9390728e2a14088236d8fe6f82635f5c84418fe37a04457e00, name=ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-3-deactivate, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.vendor=CentOS, ceph=True, RELEASE=wip-guits-main-2024-09-20-2310-f36283e, CEPH_POINT_RELEASE=, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0) 2024-09-21T04:25:19.380 INFO:journalctl@ceph.osd.3.smithi089.stdout:Sep 21 04:25:19 smithi089 podman[95003]: 2024-09-21 04:25:19.29830024 +0000 UTC m=+0.335204268 container start 7b65b7c2fdca480f6c719e8f48907e22ba8bc740cd4af503c2686292703a0dee (image=quay.ceph.io/ceph-ci/ceph@sha256:2195e5f178be0a9390728e2a14088236d8fe6f82635f5c84418fe37a04457e00, name=ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-3-deactivate, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, RELEASE=wip-guits-main-2024-09-20-2310-f36283e, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, GIT_CLEAN=True, io.buildah.version=1.37.2) 2024-09-21T04:25:19.381 INFO:journalctl@ceph.osd.3.smithi089.stdout:Sep 21 04:25:19 smithi089 podman[95003]: 2024-09-21 04:25:19.377672555 +0000 UTC m=+0.414576590 container attach 7b65b7c2fdca480f6c719e8f48907e22ba8bc740cd4af503c2686292703a0dee (image=quay.ceph.io/ceph-ci/ceph@sha256:2195e5f178be0a9390728e2a14088236d8fe6f82635f5c84418fe37a04457e00, name=ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-3-deactivate, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, ceph=True, RELEASE=wip-guits-main-2024-09-20-2310-f36283e, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=) 2024-09-21T04:25:19.522 INFO:journalctl@ceph.osd.5.smithi195.stdout:Sep 21 04:25:19 smithi195 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-5[55460]: 2024-09-21T04:25:19.260+0000 7f798bac3640 -1 osd.5 165 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2024-09-21T04:24:52.338696+0000 front 2024-09-21T04:24:52.338577+0000 (oldest deadline 2024-09-21T04:25:18.238521+0000) 2024-09-21T04:25:19.765 INFO:journalctl@ceph.osd.3.smithi089.stdout:Sep 21 04:25:19 smithi089 podman[95003]: 2024-09-21 04:25:19.48911206 +0000 UTC m=+0.526016098 container died 7b65b7c2fdca480f6c719e8f48907e22ba8bc740cd4af503c2686292703a0dee (image=quay.ceph.io/ceph-ci/ceph@sha256:2195e5f178be0a9390728e2a14088236d8fe6f82635f5c84418fe37a04457e00, name=ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-3-deactivate, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=wip-guits-main-2024-09-20-2310-f36283e, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, ceph=True, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, org.label-schema.license=GPLv2) 2024-09-21T04:25:19.859 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 21 04:25:19 smithi195 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-7[59664]: 2024-09-21T04:25:19.641+0000 7f99a543c640 -1 osd.7 165 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2024-09-21T04:24:52.054834+0000 front 2024-09-21T04:24:52.054715+0000 (oldest deadline 2024-09-21T04:25:17.855130+0000) 2024-09-21T04:25:19.859 INFO:journalctl@ceph.osd.4.smithi195.stdout:Sep 21 04:25:19 smithi195 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-4[53378]: 2024-09-21T04:25:19.519+0000 7fda08ec7640 -1 osd.4 165 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2024-09-21T04:24:52.076328+0000 front 2024-09-21T04:24:52.076253+0000 (oldest deadline 2024-09-21T04:25:17.976096+0000) 2024-09-21T04:25:19.859 INFO:journalctl@ceph.osd.6.smithi195.stdout:Sep 21 04:25:19 smithi195 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-6[57554]: 2024-09-21T04:25:19.755+0000 7f9870ab3640 -1 osd.6 165 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2024-09-21T04:24:52.395503+0000 front 2024-09-21T04:24:52.395593+0000 (oldest deadline 2024-09-21T04:25:18.295679+0000) 2024-09-21T04:25:20.025 INFO:journalctl@ceph.osd.3.smithi089.stdout:Sep 21 04:25:20 smithi089 podman[95026]: 2024-09-21 04:25:20.006126722 +0000 UTC m=+0.511382638 container remove 7b65b7c2fdca480f6c719e8f48907e22ba8bc740cd4af503c2686292703a0dee (image=quay.ceph.io/ceph-ci/ceph@sha256:2195e5f178be0a9390728e2a14088236d8fe6f82635f5c84418fe37a04457e00, name=ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-3-deactivate, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, ceph=True, RELEASE=wip-guits-main-2024-09-20-2310-f36283e) 2024-09-21T04:25:20.030 DEBUG:teuthology.orchestra.run.smithi089:> sudo pkill -f 'journalctl -f -n 0 -u ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490@osd.3.service' 2024-09-21T04:25:20.069 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-21T04:25:20.069 INFO:tasks.cephadm.osd.3:Stopped osd.3 2024-09-21T04:25:20.069 INFO:tasks.cephadm.osd.4:Stopping osd.4... 2024-09-21T04:25:20.069 DEBUG:teuthology.orchestra.run.smithi195:> sudo systemctl stop ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490@osd.4 2024-09-21T04:25:20.359 INFO:journalctl@ceph.osd.4.smithi195.stdout:Sep 21 04:25:20 smithi195 systemd[1]: Stopping Ceph osd.4 for 1b9b79fc-77ce-11ef-baf5-efdc52797490... 2024-09-21T04:25:20.359 INFO:journalctl@ceph.osd.5.smithi195.stdout:Sep 21 04:25:20 smithi195 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-5[55460]: 2024-09-21T04:25:20.261+0000 7f798bac3640 -1 osd.5 165 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2024-09-21T04:24:52.338696+0000 front 2024-09-21T04:24:52.338577+0000 (oldest deadline 2024-09-21T04:25:18.238521+0000) 2024-09-21T04:25:20.859 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 21 04:25:20 smithi195 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-7[59664]: 2024-09-21T04:25:20.608+0000 7f99a543c640 -1 osd.7 165 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2024-09-21T04:24:52.054834+0000 front 2024-09-21T04:24:52.054715+0000 (oldest deadline 2024-09-21T04:25:17.855130+0000) 2024-09-21T04:25:20.859 INFO:journalctl@ceph.osd.6.smithi195.stdout:Sep 21 04:25:20 smithi195 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-6[57554]: 2024-09-21T04:25:20.749+0000 7f9870ab3640 -1 osd.6 165 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2024-09-21T04:24:52.395503+0000 front 2024-09-21T04:24:52.395593+0000 (oldest deadline 2024-09-21T04:25:18.295679+0000) 2024-09-21T04:25:20.859 INFO:journalctl@ceph.osd.4.smithi195.stdout:Sep 21 04:25:20 smithi195 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-4[53378]: 2024-09-21T04:25:20.538+0000 7fda08ec7640 -1 osd.4 165 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2024-09-21T04:24:52.076328+0000 front 2024-09-21T04:24:52.076253+0000 (oldest deadline 2024-09-21T04:25:17.976096+0000) 2024-09-21T04:25:20.859 INFO:journalctl@ceph.osd.4.smithi195.stdout:Sep 21 04:25:20 smithi195 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-4[53378]: 2024-09-21T04:25:20.623+0000 7fda0c8b2640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.4 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-21T04:25:20.860 INFO:journalctl@ceph.osd.4.smithi195.stdout:Sep 21 04:25:20 smithi195 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-4[53378]: 2024-09-21T04:25:20.623+0000 7fda0c8b2640 -1 osd.4 165 *** Got signal Terminated *** 2024-09-21T04:25:20.860 INFO:journalctl@ceph.osd.4.smithi195.stdout:Sep 21 04:25:20 smithi195 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-4[53378]: 2024-09-21T04:25:20.623+0000 7fda0c8b2640 -1 osd.4 165 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-21T04:25:21.568 INFO:journalctl@ceph.osd.5.smithi195.stdout:Sep 21 04:25:21 smithi195 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-5[55460]: 2024-09-21T04:25:21.292+0000 7f798bac3640 -1 osd.5 165 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2024-09-21T04:24:52.338696+0000 front 2024-09-21T04:24:52.338577+0000 (oldest deadline 2024-09-21T04:25:18.238521+0000) 2024-09-21T04:25:21.859 INFO:journalctl@ceph.osd.6.smithi195.stdout:Sep 21 04:25:21 smithi195 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-6[57554]: 2024-09-21T04:25:21.704+0000 7f9870ab3640 -1 osd.6 165 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2024-09-21T04:24:52.395503+0000 front 2024-09-21T04:24:52.395593+0000 (oldest deadline 2024-09-21T04:25:18.295679+0000) 2024-09-21T04:25:21.859 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 21 04:25:21 smithi195 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-7[59664]: 2024-09-21T04:25:21.581+0000 7f99a543c640 -1 osd.7 165 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2024-09-21T04:24:52.054834+0000 front 2024-09-21T04:24:52.054715+0000 (oldest deadline 2024-09-21T04:25:17.855130+0000) 2024-09-21T04:25:21.859 INFO:journalctl@ceph.osd.4.smithi195.stdout:Sep 21 04:25:21 smithi195 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-4[53378]: 2024-09-21T04:25:21.566+0000 7fda08ec7640 -1 osd.4 165 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2024-09-21T04:24:52.076328+0000 front 2024-09-21T04:24:52.076253+0000 (oldest deadline 2024-09-21T04:25:17.976096+0000) 2024-09-21T04:25:22.609 INFO:journalctl@ceph.osd.5.smithi195.stdout:Sep 21 04:25:22 smithi195 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-5[55460]: 2024-09-21T04:25:22.327+0000 7f798bac3640 -1 osd.5 165 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2024-09-21T04:24:52.338696+0000 front 2024-09-21T04:24:52.338577+0000 (oldest deadline 2024-09-21T04:25:18.238521+0000) 2024-09-21T04:25:22.610 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 21 04:25:22 smithi195 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-7[59664]: 2024-09-21T04:25:22.559+0000 7f99a543c640 -1 osd.7 165 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2024-09-21T04:24:52.054834+0000 front 2024-09-21T04:24:52.054715+0000 (oldest deadline 2024-09-21T04:25:17.855130+0000) 2024-09-21T04:25:23.109 INFO:journalctl@ceph.osd.6.smithi195.stdout:Sep 21 04:25:22 smithi195 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-6[57554]: 2024-09-21T04:25:22.723+0000 7f9870ab3640 -1 osd.6 165 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2024-09-21T04:24:52.395503+0000 front 2024-09-21T04:24:52.395593+0000 (oldest deadline 2024-09-21T04:25:18.295679+0000) 2024-09-21T04:25:23.109 INFO:journalctl@ceph.osd.4.smithi195.stdout:Sep 21 04:25:22 smithi195 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-4[53378]: 2024-09-21T04:25:22.607+0000 7fda08ec7640 -1 osd.4 165 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2024-09-21T04:24:52.076328+0000 front 2024-09-21T04:24:52.076253+0000 (oldest deadline 2024-09-21T04:25:17.976096+0000) 2024-09-21T04:25:23.609 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 21 04:25:23 smithi195 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-7[59664]: 2024-09-21T04:25:23.524+0000 7f99a543c640 -1 osd.7 165 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2024-09-21T04:24:52.054834+0000 front 2024-09-21T04:24:52.054715+0000 (oldest deadline 2024-09-21T04:25:17.855130+0000) 2024-09-21T04:25:23.609 INFO:journalctl@ceph.osd.5.smithi195.stdout:Sep 21 04:25:23 smithi195 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-5[55460]: 2024-09-21T04:25:23.306+0000 7f798bac3640 -1 osd.5 165 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2024-09-21T04:24:52.338696+0000 front 2024-09-21T04:24:52.338577+0000 (oldest deadline 2024-09-21T04:25:18.238521+0000) 2024-09-21T04:25:24.109 INFO:journalctl@ceph.osd.6.smithi195.stdout:Sep 21 04:25:23 smithi195 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-6[57554]: 2024-09-21T04:25:23.718+0000 7f9870ab3640 -1 osd.6 165 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2024-09-21T04:24:52.395503+0000 front 2024-09-21T04:24:52.395593+0000 (oldest deadline 2024-09-21T04:25:18.295679+0000) 2024-09-21T04:25:24.109 INFO:journalctl@ceph.osd.4.smithi195.stdout:Sep 21 04:25:23 smithi195 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-4[53378]: 2024-09-21T04:25:23.633+0000 7fda08ec7640 -1 osd.4 165 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2024-09-21T04:24:52.076328+0000 front 2024-09-21T04:24:52.076253+0000 (oldest deadline 2024-09-21T04:25:17.976096+0000) 2024-09-21T04:25:24.109 INFO:journalctl@ceph.osd.4.smithi195.stdout:Sep 21 04:25:23 smithi195 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-4[53378]: 2024-09-21T04:25:23.633+0000 7fda08ec7640 -1 osd.4 165 heartbeat_check: no reply from 172.21.15.89:6814 osd.1 since back 2024-09-21T04:24:57.976653+0000 front 2024-09-21T04:24:57.976607+0000 (oldest deadline 2024-09-21T04:25:23.276460+0000) 2024-09-21T04:25:24.609 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 21 04:25:24 smithi195 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-7[59664]: 2024-09-21T04:25:24.557+0000 7f99a543c640 -1 osd.7 165 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2024-09-21T04:24:52.054834+0000 front 2024-09-21T04:24:52.054715+0000 (oldest deadline 2024-09-21T04:25:17.855130+0000) 2024-09-21T04:25:24.609 INFO:journalctl@ceph.osd.5.smithi195.stdout:Sep 21 04:25:24 smithi195 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-5[55460]: 2024-09-21T04:25:24.340+0000 7f798bac3640 -1 osd.5 165 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2024-09-21T04:24:52.338696+0000 front 2024-09-21T04:24:52.338577+0000 (oldest deadline 2024-09-21T04:25:18.238521+0000) 2024-09-21T04:25:24.609 INFO:journalctl@ceph.osd.5.smithi195.stdout:Sep 21 04:25:24 smithi195 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-5[55460]: 2024-09-21T04:25:24.340+0000 7f798bac3640 -1 osd.5 165 heartbeat_check: no reply from 172.21.15.89:6814 osd.1 since back 2024-09-21T04:25:01.739624+0000 front 2024-09-21T04:25:01.739608+0000 (oldest deadline 2024-09-21T04:25:23.439451+0000) 2024-09-21T04:25:25.109 INFO:journalctl@ceph.osd.6.smithi195.stdout:Sep 21 04:25:24 smithi195 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-6[57554]: 2024-09-21T04:25:24.760+0000 7f9870ab3640 -1 osd.6 165 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2024-09-21T04:24:52.395503+0000 front 2024-09-21T04:24:52.395593+0000 (oldest deadline 2024-09-21T04:25:18.295679+0000) 2024-09-21T04:25:25.109 INFO:journalctl@ceph.osd.4.smithi195.stdout:Sep 21 04:25:24 smithi195 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-4[53378]: 2024-09-21T04:25:24.646+0000 7fda08ec7640 -1 osd.4 165 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2024-09-21T04:24:52.076328+0000 front 2024-09-21T04:24:52.076253+0000 (oldest deadline 2024-09-21T04:25:17.976096+0000) 2024-09-21T04:25:25.109 INFO:journalctl@ceph.osd.4.smithi195.stdout:Sep 21 04:25:24 smithi195 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-4[53378]: 2024-09-21T04:25:24.646+0000 7fda08ec7640 -1 osd.4 165 heartbeat_check: no reply from 172.21.15.89:6814 osd.1 since back 2024-09-21T04:24:57.976653+0000 front 2024-09-21T04:24:57.976607+0000 (oldest deadline 2024-09-21T04:25:23.276460+0000) 2024-09-21T04:25:25.601 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 21 04:25:25 smithi195 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-7[59664]: 2024-09-21T04:25:25.533+0000 7f99a543c640 -1 osd.7 165 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2024-09-21T04:24:52.054834+0000 front 2024-09-21T04:24:52.054715+0000 (oldest deadline 2024-09-21T04:25:17.855130+0000) 2024-09-21T04:25:25.601 INFO:journalctl@ceph.osd.5.smithi195.stdout:Sep 21 04:25:25 smithi195 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-5[55460]: 2024-09-21T04:25:25.322+0000 7f798bac3640 -1 osd.5 165 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2024-09-21T04:24:52.338696+0000 front 2024-09-21T04:24:52.338577+0000 (oldest deadline 2024-09-21T04:25:18.238521+0000) 2024-09-21T04:25:25.601 INFO:journalctl@ceph.osd.5.smithi195.stdout:Sep 21 04:25:25 smithi195 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-5[55460]: 2024-09-21T04:25:25.322+0000 7f798bac3640 -1 osd.5 165 heartbeat_check: no reply from 172.21.15.89:6814 osd.1 since back 2024-09-21T04:25:01.739624+0000 front 2024-09-21T04:25:01.739608+0000 (oldest deadline 2024-09-21T04:25:23.439451+0000) 2024-09-21T04:25:25.859 INFO:journalctl@ceph.osd.6.smithi195.stdout:Sep 21 04:25:25 smithi195 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-6[57554]: 2024-09-21T04:25:25.723+0000 7f9870ab3640 -1 osd.6 165 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2024-09-21T04:24:52.395503+0000 front 2024-09-21T04:24:52.395593+0000 (oldest deadline 2024-09-21T04:25:18.295679+0000) 2024-09-21T04:25:25.859 INFO:journalctl@ceph.osd.6.smithi195.stdout:Sep 21 04:25:25 smithi195 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-6[57554]: 2024-09-21T04:25:25.723+0000 7f9870ab3640 -1 osd.6 165 heartbeat_check: no reply from 172.21.15.89:6814 osd.1 since back 2024-09-21T04:25:01.796546+0000 front 2024-09-21T04:25:01.796603+0000 (oldest deadline 2024-09-21T04:25:25.296494+0000) 2024-09-21T04:25:25.859 INFO:journalctl@ceph.osd.4.smithi195.stdout:Sep 21 04:25:25 smithi195 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-4[53378]: 2024-09-21T04:25:25.598+0000 7fda08ec7640 -1 osd.4 165 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2024-09-21T04:24:52.076328+0000 front 2024-09-21T04:24:52.076253+0000 (oldest deadline 2024-09-21T04:25:17.976096+0000) 2024-09-21T04:25:25.860 INFO:journalctl@ceph.osd.4.smithi195.stdout:Sep 21 04:25:25 smithi195 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-4[53378]: 2024-09-21T04:25:25.598+0000 7fda08ec7640 -1 osd.4 165 heartbeat_check: no reply from 172.21.15.89:6814 osd.1 since back 2024-09-21T04:24:57.976653+0000 front 2024-09-21T04:24:57.976607+0000 (oldest deadline 2024-09-21T04:25:23.276460+0000) 2024-09-21T04:25:26.485 INFO:journalctl@ceph.osd.4.smithi195.stdout:Sep 21 04:25:26 smithi195 podman[69058]: 2024-09-21 04:25:26.214700783 +0000 UTC m=+5.951137810 container died e375f90f95c434416b24612ad95401ed874031a7b56dcd8d1dfb9e29dcb09e68 (image=quay.ceph.io/ceph-ci/ceph@sha256:2195e5f178be0a9390728e2a14088236d8fe6f82635f5c84418fe37a04457e00, name=ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-4, GIT_CLEAN=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-guits-main-2024-09-20-2310-f36283e, org.label-schema.vendor=CentOS) 2024-09-21T04:25:26.485 INFO:journalctl@ceph.osd.5.smithi195.stdout:Sep 21 04:25:26 smithi195 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-5[55460]: 2024-09-21T04:25:26.354+0000 7f798bac3640 -1 osd.5 165 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2024-09-21T04:24:52.338696+0000 front 2024-09-21T04:24:52.338577+0000 (oldest deadline 2024-09-21T04:25:18.238521+0000) 2024-09-21T04:25:26.485 INFO:journalctl@ceph.osd.5.smithi195.stdout:Sep 21 04:25:26 smithi195 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-5[55460]: 2024-09-21T04:25:26.354+0000 7f798bac3640 -1 osd.5 165 heartbeat_check: no reply from 172.21.15.89:6814 osd.1 since back 2024-09-21T04:25:01.739624+0000 front 2024-09-21T04:25:01.739608+0000 (oldest deadline 2024-09-21T04:25:23.439451+0000) 2024-09-21T04:25:26.769 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 21 04:25:26 smithi195 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-7[59664]: 2024-09-21T04:25:26.524+0000 7f99a543c640 -1 osd.7 165 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2024-09-21T04:24:52.054834+0000 front 2024-09-21T04:24:52.054715+0000 (oldest deadline 2024-09-21T04:25:17.855130+0000) 2024-09-21T04:25:26.769 INFO:journalctl@ceph.osd.4.smithi195.stdout:Sep 21 04:25:26 smithi195 podman[69058]: 2024-09-21 04:25:26.648681321 +0000 UTC m=+6.385118340 container cleanup e375f90f95c434416b24612ad95401ed874031a7b56dcd8d1dfb9e29dcb09e68 (image=quay.ceph.io/ceph-ci/ceph@sha256:2195e5f178be0a9390728e2a14088236d8fe6f82635f5c84418fe37a04457e00, name=ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-4, RELEASE=wip-guits-main-2024-09-20-2310-f36283e, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, ceph=True, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-21T04:25:26.769 INFO:journalctl@ceph.osd.4.smithi195.stdout:Sep 21 04:25:26 smithi195 bash[69058]: ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-4 2024-09-21T04:25:27.109 INFO:journalctl@ceph.osd.4.smithi195.stdout:Sep 21 04:25:26 smithi195 podman[69074]: 2024-09-21 04:25:26.816706923 +0000 UTC m=+0.603668614 container remove e375f90f95c434416b24612ad95401ed874031a7b56dcd8d1dfb9e29dcb09e68 (image=quay.ceph.io/ceph-ci/ceph@sha256:2195e5f178be0a9390728e2a14088236d8fe6f82635f5c84418fe37a04457e00, name=ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-4, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, ceph=True, io.buildah.version=1.37.2, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, RELEASE=wip-guits-main-2024-09-20-2310-f36283e, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD) 2024-09-21T04:25:27.109 INFO:journalctl@ceph.osd.6.smithi195.stdout:Sep 21 04:25:26 smithi195 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-6[57554]: 2024-09-21T04:25:26.765+0000 7f9870ab3640 -1 osd.6 165 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2024-09-21T04:24:52.395503+0000 front 2024-09-21T04:24:52.395593+0000 (oldest deadline 2024-09-21T04:25:18.295679+0000) 2024-09-21T04:25:27.109 INFO:journalctl@ceph.osd.6.smithi195.stdout:Sep 21 04:25:26 smithi195 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-6[57554]: 2024-09-21T04:25:26.765+0000 7f9870ab3640 -1 osd.6 165 heartbeat_check: no reply from 172.21.15.89:6814 osd.1 since back 2024-09-21T04:25:01.796546+0000 front 2024-09-21T04:25:01.796603+0000 (oldest deadline 2024-09-21T04:25:25.296494+0000) 2024-09-21T04:25:27.572 INFO:journalctl@ceph.osd.5.smithi195.stdout:Sep 21 04:25:27 smithi195 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-5[55460]: 2024-09-21T04:25:27.328+0000 7f798bac3640 -1 osd.5 165 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2024-09-21T04:24:52.338696+0000 front 2024-09-21T04:24:52.338577+0000 (oldest deadline 2024-09-21T04:25:18.238521+0000) 2024-09-21T04:25:27.573 INFO:journalctl@ceph.osd.5.smithi195.stdout:Sep 21 04:25:27 smithi195 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-5[55460]: 2024-09-21T04:25:27.328+0000 7f798bac3640 -1 osd.5 165 heartbeat_check: no reply from 172.21.15.89:6814 osd.1 since back 2024-09-21T04:25:01.739624+0000 front 2024-09-21T04:25:01.739608+0000 (oldest deadline 2024-09-21T04:25:23.439451+0000) 2024-09-21T04:25:27.573 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 21 04:25:27 smithi195 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-7[59664]: 2024-09-21T04:25:27.505+0000 7f99a543c640 -1 osd.7 165 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2024-09-21T04:24:52.054834+0000 front 2024-09-21T04:24:52.054715+0000 (oldest deadline 2024-09-21T04:25:17.855130+0000) 2024-09-21T04:25:27.573 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 21 04:25:27 smithi195 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-7[59664]: 2024-09-21T04:25:27.505+0000 7f99a543c640 -1 osd.7 165 heartbeat_check: no reply from 172.21.15.89:6814 osd.1 since back 2024-09-21T04:25:01.356285+0000 front 2024-09-21T04:25:01.356363+0000 (oldest deadline 2024-09-21T04:25:26.556666+0000) 2024-09-21T04:25:27.573 INFO:journalctl@ceph.osd.4.smithi195.stdout:Sep 21 04:25:27 smithi195 podman[69164]: 2024-09-21 04:25:27.161488214 +0000 UTC m=+0.015582226 image pull 48ed0a2cf89d6d0371a2c6185a5869a747265f6abd5ac2a74f00cc32d5628721 quay.ceph.io/ceph-ci/ceph@sha256:2195e5f178be0a9390728e2a14088236d8fe6f82635f5c84418fe37a04457e00 2024-09-21T04:25:27.573 INFO:journalctl@ceph.osd.4.smithi195.stdout:Sep 21 04:25:27 smithi195 podman[69164]: 2024-09-21 04:25:27.275895057 +0000 UTC m=+0.129989041 container create ff14860ca831b17bfb294a8cf096e80168aaf83ff6f3842ae6e93e8d5dbcb0a3 (image=quay.ceph.io/ceph-ci/ceph@sha256:2195e5f178be0a9390728e2a14088236d8fe6f82635f5c84418fe37a04457e00, name=ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-4-deactivate, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, ceph=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, GIT_CLEAN=True, org.label-schema.license=GPLv2, RELEASE=wip-guits-main-2024-09-20-2310-f36283e, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux ) 2024-09-21T04:25:27.573 INFO:journalctl@ceph.osd.4.smithi195.stdout:Sep 21 04:25:27 smithi195 podman[69164]: 2024-09-21 04:25:27.503641608 +0000 UTC m=+0.357735612 container init ff14860ca831b17bfb294a8cf096e80168aaf83ff6f3842ae6e93e8d5dbcb0a3 (image=quay.ceph.io/ceph-ci/ceph@sha256:2195e5f178be0a9390728e2a14088236d8fe6f82635f5c84418fe37a04457e00, name=ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-4-deactivate, RELEASE=wip-guits-main-2024-09-20-2310-f36283e, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, ceph=True, maintainer=Guillaume Abrioux , GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=) 2024-09-21T04:25:27.573 INFO:journalctl@ceph.osd.4.smithi195.stdout:Sep 21 04:25:27 smithi195 podman[69164]: 2024-09-21 04:25:27.506898338 +0000 UTC m=+0.360992332 container start ff14860ca831b17bfb294a8cf096e80168aaf83ff6f3842ae6e93e8d5dbcb0a3 (image=quay.ceph.io/ceph-ci/ceph@sha256:2195e5f178be0a9390728e2a14088236d8fe6f82635f5c84418fe37a04457e00, name=ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-4-deactivate, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, RELEASE=wip-guits-main-2024-09-20-2310-f36283e, org.label-schema.license=GPLv2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_CLEAN=True, maintainer=Guillaume Abrioux , ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS) 2024-09-21T04:25:27.573 INFO:journalctl@ceph.osd.4.smithi195.stdout:Sep 21 04:25:27 smithi195 podman[69164]: 2024-09-21 04:25:27.57024201 +0000 UTC m=+0.424336023 container attach ff14860ca831b17bfb294a8cf096e80168aaf83ff6f3842ae6e93e8d5dbcb0a3 (image=quay.ceph.io/ceph-ci/ceph@sha256:2195e5f178be0a9390728e2a14088236d8fe6f82635f5c84418fe37a04457e00, name=ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-4-deactivate, org.label-schema.schema-version=1.0, RELEASE=wip-guits-main-2024-09-20-2310-f36283e, ceph=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, GIT_CLEAN=True) 2024-09-21T04:25:27.859 INFO:journalctl@ceph.osd.4.smithi195.stdout:Sep 21 04:25:27 smithi195 podman[69164]: 2024-09-21 04:25:27.693585391 +0000 UTC m=+0.547679394 container died ff14860ca831b17bfb294a8cf096e80168aaf83ff6f3842ae6e93e8d5dbcb0a3 (image=quay.ceph.io/ceph-ci/ceph@sha256:2195e5f178be0a9390728e2a14088236d8fe6f82635f5c84418fe37a04457e00, name=ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-4-deactivate, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.build-date=20240731, RELEASE=wip-guits-main-2024-09-20-2310-f36283e, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0) 2024-09-21T04:25:27.859 INFO:journalctl@ceph.osd.6.smithi195.stdout:Sep 21 04:25:27 smithi195 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-6[57554]: 2024-09-21T04:25:27.801+0000 7f9870ab3640 -1 osd.6 165 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2024-09-21T04:24:52.395503+0000 front 2024-09-21T04:24:52.395593+0000 (oldest deadline 2024-09-21T04:25:18.295679+0000) 2024-09-21T04:25:27.859 INFO:journalctl@ceph.osd.6.smithi195.stdout:Sep 21 04:25:27 smithi195 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-6[57554]: 2024-09-21T04:25:27.801+0000 7f9870ab3640 -1 osd.6 165 heartbeat_check: no reply from 172.21.15.89:6814 osd.1 since back 2024-09-21T04:25:01.796546+0000 front 2024-09-21T04:25:01.796603+0000 (oldest deadline 2024-09-21T04:25:25.296494+0000) 2024-09-21T04:25:28.291 DEBUG:teuthology.orchestra.run.smithi195:> sudo pkill -f 'journalctl -f -n 0 -u ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490@osd.4.service' 2024-09-21T04:25:28.330 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-21T04:25:28.330 INFO:tasks.cephadm.osd.4:Stopped osd.4 2024-09-21T04:25:28.330 INFO:tasks.cephadm.osd.5:Stopping osd.5... 2024-09-21T04:25:28.330 DEBUG:teuthology.orchestra.run.smithi195:> sudo systemctl stop ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490@osd.5 2024-09-21T04:25:28.521 INFO:journalctl@ceph.osd.5.smithi195.stdout:Sep 21 04:25:28 smithi195 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-5[55460]: 2024-09-21T04:25:28.370+0000 7f798bac3640 -1 osd.5 165 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2024-09-21T04:24:52.338696+0000 front 2024-09-21T04:24:52.338577+0000 (oldest deadline 2024-09-21T04:25:18.238521+0000) 2024-09-21T04:25:28.521 INFO:journalctl@ceph.osd.5.smithi195.stdout:Sep 21 04:25:28 smithi195 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-5[55460]: 2024-09-21T04:25:28.370+0000 7f798bac3640 -1 osd.5 165 heartbeat_check: no reply from 172.21.15.89:6814 osd.1 since back 2024-09-21T04:25:01.739624+0000 front 2024-09-21T04:25:01.739608+0000 (oldest deadline 2024-09-21T04:25:23.439451+0000) 2024-09-21T04:25:28.521 INFO:journalctl@ceph.osd.5.smithi195.stdout:Sep 21 04:25:28 smithi195 systemd[1]: Stopping Ceph osd.5 for 1b9b79fc-77ce-11ef-baf5-efdc52797490... 2024-09-21T04:25:28.522 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 21 04:25:28 smithi195 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-7[59664]: 2024-09-21T04:25:28.504+0000 7f99a543c640 -1 osd.7 165 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2024-09-21T04:24:52.054834+0000 front 2024-09-21T04:24:52.054715+0000 (oldest deadline 2024-09-21T04:25:17.855130+0000) 2024-09-21T04:25:28.522 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 21 04:25:28 smithi195 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-7[59664]: 2024-09-21T04:25:28.504+0000 7f99a543c640 -1 osd.7 165 heartbeat_check: no reply from 172.21.15.89:6814 osd.1 since back 2024-09-21T04:25:01.356285+0000 front 2024-09-21T04:25:01.356363+0000 (oldest deadline 2024-09-21T04:25:26.556666+0000) 2024-09-21T04:25:28.787 INFO:journalctl@ceph.osd.5.smithi195.stdout:Sep 21 04:25:28 smithi195 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-5[55460]: 2024-09-21T04:25:28.669+0000 7f798f4ae640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.5 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-21T04:25:28.787 INFO:journalctl@ceph.osd.5.smithi195.stdout:Sep 21 04:25:28 smithi195 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-5[55460]: 2024-09-21T04:25:28.669+0000 7f798f4ae640 -1 osd.5 165 *** Got signal Terminated *** 2024-09-21T04:25:28.787 INFO:journalctl@ceph.osd.5.smithi195.stdout:Sep 21 04:25:28 smithi195 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-5[55460]: 2024-09-21T04:25:28.669+0000 7f798f4ae640 -1 osd.5 165 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-21T04:25:29.109 INFO:journalctl@ceph.osd.6.smithi195.stdout:Sep 21 04:25:28 smithi195 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-6[57554]: 2024-09-21T04:25:28.784+0000 7f9870ab3640 -1 osd.6 165 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2024-09-21T04:24:52.395503+0000 front 2024-09-21T04:24:52.395593+0000 (oldest deadline 2024-09-21T04:25:18.295679+0000) 2024-09-21T04:25:29.109 INFO:journalctl@ceph.osd.6.smithi195.stdout:Sep 21 04:25:28 smithi195 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-6[57554]: 2024-09-21T04:25:28.784+0000 7f9870ab3640 -1 osd.6 165 heartbeat_check: no reply from 172.21.15.89:6814 osd.1 since back 2024-09-21T04:25:01.796546+0000 front 2024-09-21T04:25:01.796603+0000 (oldest deadline 2024-09-21T04:25:25.296494+0000) 2024-09-21T04:25:29.780 INFO:journalctl@ceph.osd.5.smithi195.stdout:Sep 21 04:25:29 smithi195 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-5[55460]: 2024-09-21T04:25:29.370+0000 7f798bac3640 -1 osd.5 165 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2024-09-21T04:24:52.338696+0000 front 2024-09-21T04:24:52.338577+0000 (oldest deadline 2024-09-21T04:25:18.238521+0000) 2024-09-21T04:25:29.780 INFO:journalctl@ceph.osd.5.smithi195.stdout:Sep 21 04:25:29 smithi195 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-5[55460]: 2024-09-21T04:25:29.370+0000 7f798bac3640 -1 osd.5 165 heartbeat_check: no reply from 172.21.15.89:6814 osd.1 since back 2024-09-21T04:25:01.739624+0000 front 2024-09-21T04:25:01.739608+0000 (oldest deadline 2024-09-21T04:25:23.439451+0000) 2024-09-21T04:25:29.781 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 21 04:25:29 smithi195 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-7[59664]: 2024-09-21T04:25:29.468+0000 7f99a543c640 -1 osd.7 165 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2024-09-21T04:24:52.054834+0000 front 2024-09-21T04:24:52.054715+0000 (oldest deadline 2024-09-21T04:25:17.855130+0000) 2024-09-21T04:25:29.781 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 21 04:25:29 smithi195 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-7[59664]: 2024-09-21T04:25:29.468+0000 7f99a543c640 -1 osd.7 165 heartbeat_check: no reply from 172.21.15.89:6814 osd.1 since back 2024-09-21T04:25:01.356285+0000 front 2024-09-21T04:25:01.356363+0000 (oldest deadline 2024-09-21T04:25:26.556666+0000) 2024-09-21T04:25:30.109 INFO:journalctl@ceph.osd.6.smithi195.stdout:Sep 21 04:25:29 smithi195 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-6[57554]: 2024-09-21T04:25:29.777+0000 7f9870ab3640 -1 osd.6 165 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2024-09-21T04:24:52.395503+0000 front 2024-09-21T04:24:52.395593+0000 (oldest deadline 2024-09-21T04:25:18.295679+0000) 2024-09-21T04:25:30.109 INFO:journalctl@ceph.osd.6.smithi195.stdout:Sep 21 04:25:29 smithi195 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-6[57554]: 2024-09-21T04:25:29.777+0000 7f9870ab3640 -1 osd.6 165 heartbeat_check: no reply from 172.21.15.89:6814 osd.1 since back 2024-09-21T04:25:01.796546+0000 front 2024-09-21T04:25:01.796603+0000 (oldest deadline 2024-09-21T04:25:25.296494+0000) 2024-09-21T04:25:30.609 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 21 04:25:30 smithi195 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-7[59664]: 2024-09-21T04:25:30.516+0000 7f99a543c640 -1 osd.7 165 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2024-09-21T04:24:52.054834+0000 front 2024-09-21T04:24:52.054715+0000 (oldest deadline 2024-09-21T04:25:17.855130+0000) 2024-09-21T04:25:30.609 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 21 04:25:30 smithi195 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-7[59664]: 2024-09-21T04:25:30.516+0000 7f99a543c640 -1 osd.7 165 heartbeat_check: no reply from 172.21.15.89:6814 osd.1 since back 2024-09-21T04:25:01.356285+0000 front 2024-09-21T04:25:01.356363+0000 (oldest deadline 2024-09-21T04:25:26.556666+0000) 2024-09-21T04:25:30.609 INFO:journalctl@ceph.osd.5.smithi195.stdout:Sep 21 04:25:30 smithi195 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-5[55460]: 2024-09-21T04:25:30.338+0000 7f798bac3640 -1 osd.5 165 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2024-09-21T04:24:52.338696+0000 front 2024-09-21T04:24:52.338577+0000 (oldest deadline 2024-09-21T04:25:18.238521+0000) 2024-09-21T04:25:30.609 INFO:journalctl@ceph.osd.5.smithi195.stdout:Sep 21 04:25:30 smithi195 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-5[55460]: 2024-09-21T04:25:30.338+0000 7f798bac3640 -1 osd.5 165 heartbeat_check: no reply from 172.21.15.89:6814 osd.1 since back 2024-09-21T04:25:01.739624+0000 front 2024-09-21T04:25:01.739608+0000 (oldest deadline 2024-09-21T04:25:23.439451+0000) 2024-09-21T04:25:31.109 INFO:journalctl@ceph.osd.6.smithi195.stdout:Sep 21 04:25:30 smithi195 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-6[57554]: 2024-09-21T04:25:30.739+0000 7f9870ab3640 -1 osd.6 165 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2024-09-21T04:24:52.395503+0000 front 2024-09-21T04:24:52.395593+0000 (oldest deadline 2024-09-21T04:25:18.295679+0000) 2024-09-21T04:25:31.109 INFO:journalctl@ceph.osd.6.smithi195.stdout:Sep 21 04:25:30 smithi195 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-6[57554]: 2024-09-21T04:25:30.739+0000 7f9870ab3640 -1 osd.6 165 heartbeat_check: no reply from 172.21.15.89:6814 osd.1 since back 2024-09-21T04:25:01.796546+0000 front 2024-09-21T04:25:01.796603+0000 (oldest deadline 2024-09-21T04:25:25.296494+0000) 2024-09-21T04:25:31.109 INFO:journalctl@ceph.osd.6.smithi195.stdout:Sep 21 04:25:30 smithi195 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-6[57554]: 2024-09-21T04:25:30.739+0000 7f9870ab3640 -1 osd.6 165 heartbeat_check: no reply from 172.21.15.89:6822 osd.2 since back 2024-09-21T04:25:07.597364+0000 front 2024-09-21T04:25:07.597396+0000 (oldest deadline 2024-09-21T04:25:30.497320+0000) 2024-09-21T04:25:31.725 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 21 04:25:31 smithi195 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-7[59664]: 2024-09-21T04:25:31.522+0000 7f99a543c640 -1 osd.7 165 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2024-09-21T04:24:52.054834+0000 front 2024-09-21T04:24:52.054715+0000 (oldest deadline 2024-09-21T04:25:17.855130+0000) 2024-09-21T04:25:31.726 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 21 04:25:31 smithi195 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-7[59664]: 2024-09-21T04:25:31.522+0000 7f99a543c640 -1 osd.7 165 heartbeat_check: no reply from 172.21.15.89:6814 osd.1 since back 2024-09-21T04:25:01.356285+0000 front 2024-09-21T04:25:01.356363+0000 (oldest deadline 2024-09-21T04:25:26.556666+0000) 2024-09-21T04:25:31.726 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 21 04:25:31 smithi195 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-7[59664]: 2024-09-21T04:25:31.522+0000 7f99a543c640 -1 osd.7 165 heartbeat_check: no reply from 172.21.15.89:6822 osd.2 since back 2024-09-21T04:25:06.557206+0000 front 2024-09-21T04:25:06.557257+0000 (oldest deadline 2024-09-21T04:25:30.657081+0000) 2024-09-21T04:25:31.726 INFO:journalctl@ceph.osd.5.smithi195.stdout:Sep 21 04:25:31 smithi195 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-5[55460]: 2024-09-21T04:25:31.384+0000 7f798bac3640 -1 osd.5 165 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2024-09-21T04:24:52.338696+0000 front 2024-09-21T04:24:52.338577+0000 (oldest deadline 2024-09-21T04:25:18.238521+0000) 2024-09-21T04:25:31.726 INFO:journalctl@ceph.osd.5.smithi195.stdout:Sep 21 04:25:31 smithi195 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-5[55460]: 2024-09-21T04:25:31.384+0000 7f798bac3640 -1 osd.5 165 heartbeat_check: no reply from 172.21.15.89:6814 osd.1 since back 2024-09-21T04:25:01.739624+0000 front 2024-09-21T04:25:01.739608+0000 (oldest deadline 2024-09-21T04:25:23.439451+0000) 2024-09-21T04:25:32.109 INFO:journalctl@ceph.osd.6.smithi195.stdout:Sep 21 04:25:31 smithi195 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-6[57554]: 2024-09-21T04:25:31.723+0000 7f9870ab3640 -1 osd.6 165 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2024-09-21T04:24:52.395503+0000 front 2024-09-21T04:24:52.395593+0000 (oldest deadline 2024-09-21T04:25:18.295679+0000) 2024-09-21T04:25:32.109 INFO:journalctl@ceph.osd.6.smithi195.stdout:Sep 21 04:25:31 smithi195 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-6[57554]: 2024-09-21T04:25:31.723+0000 7f9870ab3640 -1 osd.6 165 heartbeat_check: no reply from 172.21.15.89:6814 osd.1 since back 2024-09-21T04:25:01.796546+0000 front 2024-09-21T04:25:01.796603+0000 (oldest deadline 2024-09-21T04:25:25.296494+0000) 2024-09-21T04:25:32.109 INFO:journalctl@ceph.osd.6.smithi195.stdout:Sep 21 04:25:31 smithi195 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-6[57554]: 2024-09-21T04:25:31.723+0000 7f9870ab3640 -1 osd.6 165 heartbeat_check: no reply from 172.21.15.89:6822 osd.2 since back 2024-09-21T04:25:07.597364+0000 front 2024-09-21T04:25:07.597396+0000 (oldest deadline 2024-09-21T04:25:30.497320+0000) 2024-09-21T04:25:32.681 INFO:journalctl@ceph.osd.5.smithi195.stdout:Sep 21 04:25:32 smithi195 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-5[55460]: 2024-09-21T04:25:32.406+0000 7f798bac3640 -1 osd.5 165 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2024-09-21T04:24:52.338696+0000 front 2024-09-21T04:24:52.338577+0000 (oldest deadline 2024-09-21T04:25:18.238521+0000) 2024-09-21T04:25:32.682 INFO:journalctl@ceph.osd.5.smithi195.stdout:Sep 21 04:25:32 smithi195 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-5[55460]: 2024-09-21T04:25:32.406+0000 7f798bac3640 -1 osd.5 165 heartbeat_check: no reply from 172.21.15.89:6814 osd.1 since back 2024-09-21T04:25:01.739624+0000 front 2024-09-21T04:25:01.739608+0000 (oldest deadline 2024-09-21T04:25:23.439451+0000) 2024-09-21T04:25:32.682 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 21 04:25:32 smithi195 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-7[59664]: 2024-09-21T04:25:32.500+0000 7f99a543c640 -1 osd.7 165 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2024-09-21T04:24:52.054834+0000 front 2024-09-21T04:24:52.054715+0000 (oldest deadline 2024-09-21T04:25:17.855130+0000) 2024-09-21T04:25:32.682 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 21 04:25:32 smithi195 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-7[59664]: 2024-09-21T04:25:32.500+0000 7f99a543c640 -1 osd.7 165 heartbeat_check: no reply from 172.21.15.89:6814 osd.1 since back 2024-09-21T04:25:01.356285+0000 front 2024-09-21T04:25:01.356363+0000 (oldest deadline 2024-09-21T04:25:26.556666+0000) 2024-09-21T04:25:32.682 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 21 04:25:32 smithi195 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-7[59664]: 2024-09-21T04:25:32.500+0000 7f99a543c640 -1 osd.7 165 heartbeat_check: no reply from 172.21.15.89:6822 osd.2 since back 2024-09-21T04:25:06.557206+0000 front 2024-09-21T04:25:06.557257+0000 (oldest deadline 2024-09-21T04:25:30.657081+0000) 2024-09-21T04:25:33.109 INFO:journalctl@ceph.osd.6.smithi195.stdout:Sep 21 04:25:32 smithi195 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-6[57554]: 2024-09-21T04:25:32.679+0000 7f9870ab3640 -1 osd.6 165 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2024-09-21T04:24:52.395503+0000 front 2024-09-21T04:24:52.395593+0000 (oldest deadline 2024-09-21T04:25:18.295679+0000) 2024-09-21T04:25:33.109 INFO:journalctl@ceph.osd.6.smithi195.stdout:Sep 21 04:25:32 smithi195 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-6[57554]: 2024-09-21T04:25:32.679+0000 7f9870ab3640 -1 osd.6 165 heartbeat_check: no reply from 172.21.15.89:6814 osd.1 since back 2024-09-21T04:25:01.796546+0000 front 2024-09-21T04:25:01.796603+0000 (oldest deadline 2024-09-21T04:25:25.296494+0000) 2024-09-21T04:25:33.109 INFO:journalctl@ceph.osd.6.smithi195.stdout:Sep 21 04:25:32 smithi195 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-6[57554]: 2024-09-21T04:25:32.679+0000 7f9870ab3640 -1 osd.6 165 heartbeat_check: no reply from 172.21.15.89:6822 osd.2 since back 2024-09-21T04:25:07.597364+0000 front 2024-09-21T04:25:07.597396+0000 (oldest deadline 2024-09-21T04:25:30.497320+0000) 2024-09-21T04:25:33.859 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 21 04:25:33 smithi195 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-7[59664]: 2024-09-21T04:25:33.522+0000 7f99a543c640 -1 osd.7 165 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2024-09-21T04:24:52.054834+0000 front 2024-09-21T04:24:52.054715+0000 (oldest deadline 2024-09-21T04:25:17.855130+0000) 2024-09-21T04:25:33.859 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 21 04:25:33 smithi195 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-7[59664]: 2024-09-21T04:25:33.522+0000 7f99a543c640 -1 osd.7 165 heartbeat_check: no reply from 172.21.15.89:6814 osd.1 since back 2024-09-21T04:25:01.356285+0000 front 2024-09-21T04:25:01.356363+0000 (oldest deadline 2024-09-21T04:25:26.556666+0000) 2024-09-21T04:25:33.859 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 21 04:25:33 smithi195 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-7[59664]: 2024-09-21T04:25:33.522+0000 7f99a543c640 -1 osd.7 165 heartbeat_check: no reply from 172.21.15.89:6822 osd.2 since back 2024-09-21T04:25:06.557206+0000 front 2024-09-21T04:25:06.557257+0000 (oldest deadline 2024-09-21T04:25:30.657081+0000) 2024-09-21T04:25:33.860 INFO:journalctl@ceph.osd.5.smithi195.stdout:Sep 21 04:25:33 smithi195 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-5[55460]: 2024-09-21T04:25:33.394+0000 7f798bac3640 -1 osd.5 165 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2024-09-21T04:24:52.338696+0000 front 2024-09-21T04:24:52.338577+0000 (oldest deadline 2024-09-21T04:25:18.238521+0000) 2024-09-21T04:25:33.860 INFO:journalctl@ceph.osd.5.smithi195.stdout:Sep 21 04:25:33 smithi195 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-5[55460]: 2024-09-21T04:25:33.394+0000 7f798bac3640 -1 osd.5 165 heartbeat_check: no reply from 172.21.15.89:6814 osd.1 since back 2024-09-21T04:25:01.739624+0000 front 2024-09-21T04:25:01.739608+0000 (oldest deadline 2024-09-21T04:25:23.439451+0000) 2024-09-21T04:25:33.860 INFO:journalctl@ceph.osd.5.smithi195.stdout:Sep 21 04:25:33 smithi195 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-5[55460]: 2024-09-21T04:25:33.394+0000 7f798bac3640 -1 osd.5 165 heartbeat_check: no reply from 172.21.15.89:6822 osd.2 since back 2024-09-21T04:25:09.741474+0000 front 2024-09-21T04:25:09.741487+0000 (oldest deadline 2024-09-21T04:25:33.241481+0000) 2024-09-21T04:25:33.860 INFO:journalctl@ceph.osd.6.smithi195.stdout:Sep 21 04:25:33 smithi195 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-6[57554]: 2024-09-21T04:25:33.633+0000 7f9870ab3640 -1 osd.6 165 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2024-09-21T04:24:52.395503+0000 front 2024-09-21T04:24:52.395593+0000 (oldest deadline 2024-09-21T04:25:18.295679+0000) 2024-09-21T04:25:33.860 INFO:journalctl@ceph.osd.6.smithi195.stdout:Sep 21 04:25:33 smithi195 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-6[57554]: 2024-09-21T04:25:33.633+0000 7f9870ab3640 -1 osd.6 165 heartbeat_check: no reply from 172.21.15.89:6814 osd.1 since back 2024-09-21T04:25:01.796546+0000 front 2024-09-21T04:25:01.796603+0000 (oldest deadline 2024-09-21T04:25:25.296494+0000) 2024-09-21T04:25:33.860 INFO:journalctl@ceph.osd.6.smithi195.stdout:Sep 21 04:25:33 smithi195 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-6[57554]: 2024-09-21T04:25:33.633+0000 7f9870ab3640 -1 osd.6 165 heartbeat_check: no reply from 172.21.15.89:6822 osd.2 since back 2024-09-21T04:25:07.597364+0000 front 2024-09-21T04:25:07.597396+0000 (oldest deadline 2024-09-21T04:25:30.497320+0000) 2024-09-21T04:25:34.359 INFO:journalctl@ceph.osd.5.smithi195.stdout:Sep 21 04:25:34 smithi195 podman[69268]: 2024-09-21 04:25:34.066561812 +0000 UTC m=+5.515368599 container died 41de0beb713dabd6c82bbde19afc417f126e268c6ec14b757a87940c679e7f45 (image=quay.ceph.io/ceph-ci/ceph@sha256:2195e5f178be0a9390728e2a14088236d8fe6f82635f5c84418fe37a04457e00, name=ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-5, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, RELEASE=wip-guits-main-2024-09-20-2310-f36283e, ceph=True, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, GIT_CLEAN=True) 2024-09-21T04:25:34.619 INFO:journalctl@ceph.osd.5.smithi195.stdout:Sep 21 04:25:34 smithi195 podman[69268]: 2024-09-21 04:25:34.522568287 +0000 UTC m=+5.971375111 container cleanup 41de0beb713dabd6c82bbde19afc417f126e268c6ec14b757a87940c679e7f45 (image=quay.ceph.io/ceph-ci/ceph@sha256:2195e5f178be0a9390728e2a14088236d8fe6f82635f5c84418fe37a04457e00, name=ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-5, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, RELEASE=wip-guits-main-2024-09-20-2310-f36283e, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, ceph=True, GIT_CLEAN=True, maintainer=Guillaume Abrioux ) 2024-09-21T04:25:34.619 INFO:journalctl@ceph.osd.5.smithi195.stdout:Sep 21 04:25:34 smithi195 bash[69268]: ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-5 2024-09-21T04:25:34.620 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 21 04:25:34 smithi195 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-7[59664]: 2024-09-21T04:25:34.564+0000 7f99a543c640 -1 osd.7 165 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2024-09-21T04:24:52.054834+0000 front 2024-09-21T04:24:52.054715+0000 (oldest deadline 2024-09-21T04:25:17.855130+0000) 2024-09-21T04:25:34.620 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 21 04:25:34 smithi195 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-7[59664]: 2024-09-21T04:25:34.564+0000 7f99a543c640 -1 osd.7 165 heartbeat_check: no reply from 172.21.15.89:6814 osd.1 since back 2024-09-21T04:25:01.356285+0000 front 2024-09-21T04:25:01.356363+0000 (oldest deadline 2024-09-21T04:25:26.556666+0000) 2024-09-21T04:25:34.620 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 21 04:25:34 smithi195 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-7[59664]: 2024-09-21T04:25:34.564+0000 7f99a543c640 -1 osd.7 165 heartbeat_check: no reply from 172.21.15.89:6822 osd.2 since back 2024-09-21T04:25:06.557206+0000 front 2024-09-21T04:25:06.557257+0000 (oldest deadline 2024-09-21T04:25:30.657081+0000) 2024-09-21T04:25:35.109 INFO:journalctl@ceph.osd.5.smithi195.stdout:Sep 21 04:25:34 smithi195 podman[69312]: 2024-09-21 04:25:34.721444172 +0000 UTC m=+0.644825490 container remove 41de0beb713dabd6c82bbde19afc417f126e268c6ec14b757a87940c679e7f45 (image=quay.ceph.io/ceph-ci/ceph@sha256:2195e5f178be0a9390728e2a14088236d8fe6f82635f5c84418fe37a04457e00, name=ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-5, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-guits-main-2024-09-20-2310-f36283e, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, ceph=True, CEPH_POINT_RELEASE=) 2024-09-21T04:25:35.109 INFO:journalctl@ceph.osd.6.smithi195.stdout:Sep 21 04:25:34 smithi195 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-6[57554]: 2024-09-21T04:25:34.616+0000 7f9870ab3640 -1 osd.6 165 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2024-09-21T04:24:52.395503+0000 front 2024-09-21T04:24:52.395593+0000 (oldest deadline 2024-09-21T04:25:18.295679+0000) 2024-09-21T04:25:35.110 INFO:journalctl@ceph.osd.6.smithi195.stdout:Sep 21 04:25:34 smithi195 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-6[57554]: 2024-09-21T04:25:34.616+0000 7f9870ab3640 -1 osd.6 165 heartbeat_check: no reply from 172.21.15.89:6814 osd.1 since back 2024-09-21T04:25:01.796546+0000 front 2024-09-21T04:25:01.796603+0000 (oldest deadline 2024-09-21T04:25:25.296494+0000) 2024-09-21T04:25:35.110 INFO:journalctl@ceph.osd.6.smithi195.stdout:Sep 21 04:25:34 smithi195 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-6[57554]: 2024-09-21T04:25:34.616+0000 7f9870ab3640 -1 osd.6 165 heartbeat_check: no reply from 172.21.15.89:6822 osd.2 since back 2024-09-21T04:25:07.597364+0000 front 2024-09-21T04:25:07.597396+0000 (oldest deadline 2024-09-21T04:25:30.497320+0000) 2024-09-21T04:25:35.384 INFO:journalctl@ceph.osd.5.smithi195.stdout:Sep 21 04:25:35 smithi195 podman[69400]: 2024-09-21 04:25:35.034634149 +0000 UTC m=+0.015508422 image pull 48ed0a2cf89d6d0371a2c6185a5869a747265f6abd5ac2a74f00cc32d5628721 quay.ceph.io/ceph-ci/ceph@sha256:2195e5f178be0a9390728e2a14088236d8fe6f82635f5c84418fe37a04457e00 2024-09-21T04:25:35.384 INFO:journalctl@ceph.osd.5.smithi195.stdout:Sep 21 04:25:35 smithi195 podman[69400]: 2024-09-21 04:25:35.163811822 +0000 UTC m=+0.144686084 container create a08398f59a4b639a3bf4284060b4d10f03723a494ec000c14e88bf88521f6aea (image=quay.ceph.io/ceph-ci/ceph@sha256:2195e5f178be0a9390728e2a14088236d8fe6f82635f5c84418fe37a04457e00, name=ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-5-deactivate, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , RELEASE=wip-guits-main-2024-09-20-2310-f36283e, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, ceph=True, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731) 2024-09-21T04:25:35.385 INFO:journalctl@ceph.osd.5.smithi195.stdout:Sep 21 04:25:35 smithi195 podman[69400]: 2024-09-21 04:25:35.382642469 +0000 UTC m=+0.363516742 container init a08398f59a4b639a3bf4284060b4d10f03723a494ec000c14e88bf88521f6aea (image=quay.ceph.io/ceph-ci/ceph@sha256:2195e5f178be0a9390728e2a14088236d8fe6f82635f5c84418fe37a04457e00, name=ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-5-deactivate, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=wip-guits-main-2024-09-20-2310-f36283e, ceph=True, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-21T04:25:35.859 INFO:journalctl@ceph.osd.6.smithi195.stdout:Sep 21 04:25:35 smithi195 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-6[57554]: 2024-09-21T04:25:35.627+0000 7f9870ab3640 -1 osd.6 165 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2024-09-21T04:24:52.395503+0000 front 2024-09-21T04:24:52.395593+0000 (oldest deadline 2024-09-21T04:25:18.295679+0000) 2024-09-21T04:25:35.859 INFO:journalctl@ceph.osd.6.smithi195.stdout:Sep 21 04:25:35 smithi195 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-6[57554]: 2024-09-21T04:25:35.627+0000 7f9870ab3640 -1 osd.6 165 heartbeat_check: no reply from 172.21.15.89:6814 osd.1 since back 2024-09-21T04:25:01.796546+0000 front 2024-09-21T04:25:01.796603+0000 (oldest deadline 2024-09-21T04:25:25.296494+0000) 2024-09-21T04:25:35.859 INFO:journalctl@ceph.osd.6.smithi195.stdout:Sep 21 04:25:35 smithi195 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-6[57554]: 2024-09-21T04:25:35.627+0000 7f9870ab3640 -1 osd.6 165 heartbeat_check: no reply from 172.21.15.89:6822 osd.2 since back 2024-09-21T04:25:07.597364+0000 front 2024-09-21T04:25:07.597396+0000 (oldest deadline 2024-09-21T04:25:30.497320+0000) 2024-09-21T04:25:35.860 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 21 04:25:35 smithi195 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-7[59664]: 2024-09-21T04:25:35.575+0000 7f99a543c640 -1 osd.7 165 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2024-09-21T04:24:52.054834+0000 front 2024-09-21T04:24:52.054715+0000 (oldest deadline 2024-09-21T04:25:17.855130+0000) 2024-09-21T04:25:35.860 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 21 04:25:35 smithi195 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-7[59664]: 2024-09-21T04:25:35.575+0000 7f99a543c640 -1 osd.7 165 heartbeat_check: no reply from 172.21.15.89:6814 osd.1 since back 2024-09-21T04:25:01.356285+0000 front 2024-09-21T04:25:01.356363+0000 (oldest deadline 2024-09-21T04:25:26.556666+0000) 2024-09-21T04:25:35.860 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 21 04:25:35 smithi195 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-7[59664]: 2024-09-21T04:25:35.575+0000 7f99a543c640 -1 osd.7 165 heartbeat_check: no reply from 172.21.15.89:6822 osd.2 since back 2024-09-21T04:25:06.557206+0000 front 2024-09-21T04:25:06.557257+0000 (oldest deadline 2024-09-21T04:25:30.657081+0000) 2024-09-21T04:25:35.860 INFO:journalctl@ceph.osd.5.smithi195.stdout:Sep 21 04:25:35 smithi195 podman[69400]: 2024-09-21 04:25:35.386011671 +0000 UTC m=+0.366885935 container start a08398f59a4b639a3bf4284060b4d10f03723a494ec000c14e88bf88521f6aea (image=quay.ceph.io/ceph-ci/ceph@sha256:2195e5f178be0a9390728e2a14088236d8fe6f82635f5c84418fe37a04457e00, name=ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-5-deactivate, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=wip-guits-main-2024-09-20-2310-f36283e, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, GIT_CLEAN=True, ceph=True, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux ) 2024-09-21T04:25:35.860 INFO:journalctl@ceph.osd.5.smithi195.stdout:Sep 21 04:25:35 smithi195 podman[69400]: 2024-09-21 04:25:35.459745297 +0000 UTC m=+0.440619587 container attach a08398f59a4b639a3bf4284060b4d10f03723a494ec000c14e88bf88521f6aea (image=quay.ceph.io/ceph-ci/ceph@sha256:2195e5f178be0a9390728e2a14088236d8fe6f82635f5c84418fe37a04457e00, name=ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-5-deactivate, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.schema-version=1.0, RELEASE=wip-guits-main-2024-09-20-2310-f36283e, CEPH_POINT_RELEASE=, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, GIT_CLEAN=True, maintainer=Guillaume Abrioux ) 2024-09-21T04:25:35.860 INFO:journalctl@ceph.osd.5.smithi195.stdout:Sep 21 04:25:35 smithi195 podman[69400]: 2024-09-21 04:25:35.576831392 +0000 UTC m=+0.557705656 container died a08398f59a4b639a3bf4284060b4d10f03723a494ec000c14e88bf88521f6aea (image=quay.ceph.io/ceph-ci/ceph@sha256:2195e5f178be0a9390728e2a14088236d8fe6f82635f5c84418fe37a04457e00, name=ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-5-deactivate, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, RELEASE=wip-guits-main-2024-09-20-2310-f36283e, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2) 2024-09-21T04:25:36.288 DEBUG:teuthology.orchestra.run.smithi195:> sudo pkill -f 'journalctl -f -n 0 -u ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490@osd.5.service' 2024-09-21T04:25:36.329 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-21T04:25:36.330 INFO:tasks.cephadm.osd.5:Stopped osd.5 2024-09-21T04:25:36.330 INFO:tasks.cephadm.osd.6:Stopping osd.6... 2024-09-21T04:25:36.330 DEBUG:teuthology.orchestra.run.smithi195:> sudo systemctl stop ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490@osd.6 2024-09-21T04:25:36.593 INFO:journalctl@ceph.osd.6.smithi195.stdout:Sep 21 04:25:36 smithi195 systemd[1]: Stopping Ceph osd.6 for 1b9b79fc-77ce-11ef-baf5-efdc52797490... 2024-09-21T04:25:36.859 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 21 04:25:36 smithi195 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-7[59664]: 2024-09-21T04:25:36.589+0000 7f99a543c640 -1 osd.7 165 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2024-09-21T04:24:52.054834+0000 front 2024-09-21T04:24:52.054715+0000 (oldest deadline 2024-09-21T04:25:17.855130+0000) 2024-09-21T04:25:36.859 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 21 04:25:36 smithi195 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-7[59664]: 2024-09-21T04:25:36.589+0000 7f99a543c640 -1 osd.7 165 heartbeat_check: no reply from 172.21.15.89:6814 osd.1 since back 2024-09-21T04:25:01.356285+0000 front 2024-09-21T04:25:01.356363+0000 (oldest deadline 2024-09-21T04:25:26.556666+0000) 2024-09-21T04:25:36.859 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 21 04:25:36 smithi195 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-7[59664]: 2024-09-21T04:25:36.589+0000 7f99a543c640 -1 osd.7 165 heartbeat_check: no reply from 172.21.15.89:6822 osd.2 since back 2024-09-21T04:25:06.557206+0000 front 2024-09-21T04:25:06.557257+0000 (oldest deadline 2024-09-21T04:25:30.657081+0000) 2024-09-21T04:25:36.859 INFO:journalctl@ceph.osd.6.smithi195.stdout:Sep 21 04:25:36 smithi195 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-6[57554]: 2024-09-21T04:25:36.612+0000 7f9870ab3640 -1 osd.6 165 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2024-09-21T04:24:52.395503+0000 front 2024-09-21T04:24:52.395593+0000 (oldest deadline 2024-09-21T04:25:18.295679+0000) 2024-09-21T04:25:36.859 INFO:journalctl@ceph.osd.6.smithi195.stdout:Sep 21 04:25:36 smithi195 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-6[57554]: 2024-09-21T04:25:36.612+0000 7f9870ab3640 -1 osd.6 165 heartbeat_check: no reply from 172.21.15.89:6814 osd.1 since back 2024-09-21T04:25:01.796546+0000 front 2024-09-21T04:25:01.796603+0000 (oldest deadline 2024-09-21T04:25:25.296494+0000) 2024-09-21T04:25:36.860 INFO:journalctl@ceph.osd.6.smithi195.stdout:Sep 21 04:25:36 smithi195 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-6[57554]: 2024-09-21T04:25:36.612+0000 7f9870ab3640 -1 osd.6 165 heartbeat_check: no reply from 172.21.15.89:6822 osd.2 since back 2024-09-21T04:25:07.597364+0000 front 2024-09-21T04:25:07.597396+0000 (oldest deadline 2024-09-21T04:25:30.497320+0000) 2024-09-21T04:25:36.860 INFO:journalctl@ceph.osd.6.smithi195.stdout:Sep 21 04:25:36 smithi195 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-6[57554]: 2024-09-21T04:25:36.683+0000 7f98784a6640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.6 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-21T04:25:36.860 INFO:journalctl@ceph.osd.6.smithi195.stdout:Sep 21 04:25:36 smithi195 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-6[57554]: 2024-09-21T04:25:36.683+0000 7f98784a6640 -1 osd.6 165 *** Got signal Terminated *** 2024-09-21T04:25:36.860 INFO:journalctl@ceph.osd.6.smithi195.stdout:Sep 21 04:25:36 smithi195 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-6[57554]: 2024-09-21T04:25:36.683+0000 7f98784a6640 -1 osd.6 165 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-21T04:25:37.859 INFO:journalctl@ceph.osd.6.smithi195.stdout:Sep 21 04:25:37 smithi195 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-6[57554]: 2024-09-21T04:25:37.594+0000 7f9870ab3640 -1 osd.6 165 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2024-09-21T04:24:52.395503+0000 front 2024-09-21T04:24:52.395593+0000 (oldest deadline 2024-09-21T04:25:18.295679+0000) 2024-09-21T04:25:37.859 INFO:journalctl@ceph.osd.6.smithi195.stdout:Sep 21 04:25:37 smithi195 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-6[57554]: 2024-09-21T04:25:37.594+0000 7f9870ab3640 -1 osd.6 165 heartbeat_check: no reply from 172.21.15.89:6814 osd.1 since back 2024-09-21T04:25:01.796546+0000 front 2024-09-21T04:25:01.796603+0000 (oldest deadline 2024-09-21T04:25:25.296494+0000) 2024-09-21T04:25:37.860 INFO:journalctl@ceph.osd.6.smithi195.stdout:Sep 21 04:25:37 smithi195 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-6[57554]: 2024-09-21T04:25:37.594+0000 7f9870ab3640 -1 osd.6 165 heartbeat_check: no reply from 172.21.15.89:6822 osd.2 since back 2024-09-21T04:25:07.597364+0000 front 2024-09-21T04:25:07.597396+0000 (oldest deadline 2024-09-21T04:25:30.497320+0000) 2024-09-21T04:25:37.860 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 21 04:25:37 smithi195 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-7[59664]: 2024-09-21T04:25:37.637+0000 7f99a543c640 -1 osd.7 165 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2024-09-21T04:24:52.054834+0000 front 2024-09-21T04:24:52.054715+0000 (oldest deadline 2024-09-21T04:25:17.855130+0000) 2024-09-21T04:25:37.860 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 21 04:25:37 smithi195 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-7[59664]: 2024-09-21T04:25:37.637+0000 7f99a543c640 -1 osd.7 165 heartbeat_check: no reply from 172.21.15.89:6814 osd.1 since back 2024-09-21T04:25:01.356285+0000 front 2024-09-21T04:25:01.356363+0000 (oldest deadline 2024-09-21T04:25:26.556666+0000) 2024-09-21T04:25:37.860 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 21 04:25:37 smithi195 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-7[59664]: 2024-09-21T04:25:37.637+0000 7f99a543c640 -1 osd.7 165 heartbeat_check: no reply from 172.21.15.89:6822 osd.2 since back 2024-09-21T04:25:06.557206+0000 front 2024-09-21T04:25:06.557257+0000 (oldest deadline 2024-09-21T04:25:30.657081+0000) 2024-09-21T04:25:38.859 INFO:journalctl@ceph.osd.6.smithi195.stdout:Sep 21 04:25:38 smithi195 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-6[57554]: 2024-09-21T04:25:38.587+0000 7f9870ab3640 -1 osd.6 165 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2024-09-21T04:24:52.395503+0000 front 2024-09-21T04:24:52.395593+0000 (oldest deadline 2024-09-21T04:25:18.295679+0000) 2024-09-21T04:25:38.859 INFO:journalctl@ceph.osd.6.smithi195.stdout:Sep 21 04:25:38 smithi195 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-6[57554]: 2024-09-21T04:25:38.587+0000 7f9870ab3640 -1 osd.6 165 heartbeat_check: no reply from 172.21.15.89:6814 osd.1 since back 2024-09-21T04:25:01.796546+0000 front 2024-09-21T04:25:01.796603+0000 (oldest deadline 2024-09-21T04:25:25.296494+0000) 2024-09-21T04:25:38.859 INFO:journalctl@ceph.osd.6.smithi195.stdout:Sep 21 04:25:38 smithi195 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-6[57554]: 2024-09-21T04:25:38.587+0000 7f9870ab3640 -1 osd.6 165 heartbeat_check: no reply from 172.21.15.89:6822 osd.2 since back 2024-09-21T04:25:07.597364+0000 front 2024-09-21T04:25:07.597396+0000 (oldest deadline 2024-09-21T04:25:30.497320+0000) 2024-09-21T04:25:38.859 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 21 04:25:38 smithi195 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-7[59664]: 2024-09-21T04:25:38.638+0000 7f99a543c640 -1 osd.7 165 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2024-09-21T04:24:52.054834+0000 front 2024-09-21T04:24:52.054715+0000 (oldest deadline 2024-09-21T04:25:17.855130+0000) 2024-09-21T04:25:38.859 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 21 04:25:38 smithi195 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-7[59664]: 2024-09-21T04:25:38.638+0000 7f99a543c640 -1 osd.7 165 heartbeat_check: no reply from 172.21.15.89:6814 osd.1 since back 2024-09-21T04:25:01.356285+0000 front 2024-09-21T04:25:01.356363+0000 (oldest deadline 2024-09-21T04:25:26.556666+0000) 2024-09-21T04:25:38.860 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 21 04:25:38 smithi195 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-7[59664]: 2024-09-21T04:25:38.638+0000 7f99a543c640 -1 osd.7 165 heartbeat_check: no reply from 172.21.15.89:6822 osd.2 since back 2024-09-21T04:25:06.557206+0000 front 2024-09-21T04:25:06.557257+0000 (oldest deadline 2024-09-21T04:25:30.657081+0000) 2024-09-21T04:25:39.859 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 21 04:25:39 smithi195 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-7[59664]: 2024-09-21T04:25:39.675+0000 7f99a543c640 -1 osd.7 165 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2024-09-21T04:24:52.054834+0000 front 2024-09-21T04:24:52.054715+0000 (oldest deadline 2024-09-21T04:25:17.855130+0000) 2024-09-21T04:25:39.859 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 21 04:25:39 smithi195 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-7[59664]: 2024-09-21T04:25:39.675+0000 7f99a543c640 -1 osd.7 165 heartbeat_check: no reply from 172.21.15.89:6814 osd.1 since back 2024-09-21T04:25:01.356285+0000 front 2024-09-21T04:25:01.356363+0000 (oldest deadline 2024-09-21T04:25:26.556666+0000) 2024-09-21T04:25:39.859 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 21 04:25:39 smithi195 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-7[59664]: 2024-09-21T04:25:39.675+0000 7f99a543c640 -1 osd.7 165 heartbeat_check: no reply from 172.21.15.89:6822 osd.2 since back 2024-09-21T04:25:06.557206+0000 front 2024-09-21T04:25:06.557257+0000 (oldest deadline 2024-09-21T04:25:30.657081+0000) 2024-09-21T04:25:39.859 INFO:journalctl@ceph.osd.6.smithi195.stdout:Sep 21 04:25:39 smithi195 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-6[57554]: 2024-09-21T04:25:39.565+0000 7f9870ab3640 -1 osd.6 165 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2024-09-21T04:24:52.395503+0000 front 2024-09-21T04:24:52.395593+0000 (oldest deadline 2024-09-21T04:25:18.295679+0000) 2024-09-21T04:25:39.859 INFO:journalctl@ceph.osd.6.smithi195.stdout:Sep 21 04:25:39 smithi195 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-6[57554]: 2024-09-21T04:25:39.565+0000 7f9870ab3640 -1 osd.6 165 heartbeat_check: no reply from 172.21.15.89:6814 osd.1 since back 2024-09-21T04:25:01.796546+0000 front 2024-09-21T04:25:01.796603+0000 (oldest deadline 2024-09-21T04:25:25.296494+0000) 2024-09-21T04:25:39.860 INFO:journalctl@ceph.osd.6.smithi195.stdout:Sep 21 04:25:39 smithi195 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-6[57554]: 2024-09-21T04:25:39.566+0000 7f9870ab3640 -1 osd.6 165 heartbeat_check: no reply from 172.21.15.89:6822 osd.2 since back 2024-09-21T04:25:07.597364+0000 front 2024-09-21T04:25:07.597396+0000 (oldest deadline 2024-09-21T04:25:30.497320+0000) 2024-09-21T04:25:40.859 INFO:journalctl@ceph.osd.6.smithi195.stdout:Sep 21 04:25:40 smithi195 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-6[57554]: 2024-09-21T04:25:40.565+0000 7f9870ab3640 -1 osd.6 165 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2024-09-21T04:24:52.395503+0000 front 2024-09-21T04:24:52.395593+0000 (oldest deadline 2024-09-21T04:25:18.295679+0000) 2024-09-21T04:25:40.859 INFO:journalctl@ceph.osd.6.smithi195.stdout:Sep 21 04:25:40 smithi195 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-6[57554]: 2024-09-21T04:25:40.565+0000 7f9870ab3640 -1 osd.6 165 heartbeat_check: no reply from 172.21.15.89:6814 osd.1 since back 2024-09-21T04:25:01.796546+0000 front 2024-09-21T04:25:01.796603+0000 (oldest deadline 2024-09-21T04:25:25.296494+0000) 2024-09-21T04:25:40.859 INFO:journalctl@ceph.osd.6.smithi195.stdout:Sep 21 04:25:40 smithi195 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-6[57554]: 2024-09-21T04:25:40.565+0000 7f9870ab3640 -1 osd.6 165 heartbeat_check: no reply from 172.21.15.89:6822 osd.2 since back 2024-09-21T04:25:07.597364+0000 front 2024-09-21T04:25:07.597396+0000 (oldest deadline 2024-09-21T04:25:30.497320+0000) 2024-09-21T04:25:40.859 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 21 04:25:40 smithi195 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-7[59664]: 2024-09-21T04:25:40.715+0000 7f99a543c640 -1 osd.7 165 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2024-09-21T04:24:52.054834+0000 front 2024-09-21T04:24:52.054715+0000 (oldest deadline 2024-09-21T04:25:17.855130+0000) 2024-09-21T04:25:40.860 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 21 04:25:40 smithi195 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-7[59664]: 2024-09-21T04:25:40.715+0000 7f99a543c640 -1 osd.7 165 heartbeat_check: no reply from 172.21.15.89:6814 osd.1 since back 2024-09-21T04:25:01.356285+0000 front 2024-09-21T04:25:01.356363+0000 (oldest deadline 2024-09-21T04:25:26.556666+0000) 2024-09-21T04:25:40.860 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 21 04:25:40 smithi195 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-7[59664]: 2024-09-21T04:25:40.715+0000 7f99a543c640 -1 osd.7 165 heartbeat_check: no reply from 172.21.15.89:6822 osd.2 since back 2024-09-21T04:25:06.557206+0000 front 2024-09-21T04:25:06.557257+0000 (oldest deadline 2024-09-21T04:25:30.657081+0000) 2024-09-21T04:25:40.860 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 21 04:25:40 smithi195 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-7[59664]: 2024-09-21T04:25:40.715+0000 7f99a543c640 -1 osd.7 165 heartbeat_check: no reply from 172.21.15.89:6830 osd.3 since back 2024-09-21T04:25:15.358024+0000 front 2024-09-21T04:25:15.357954+0000 (oldest deadline 2024-09-21T04:25:39.958220+0000) 2024-09-21T04:25:42.109 INFO:journalctl@ceph.osd.6.smithi195.stdout:Sep 21 04:25:41 smithi195 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-6[57554]: 2024-09-21T04:25:41.614+0000 7f9870ab3640 -1 osd.6 165 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2024-09-21T04:24:52.395503+0000 front 2024-09-21T04:24:52.395593+0000 (oldest deadline 2024-09-21T04:25:18.295679+0000) 2024-09-21T04:25:42.109 INFO:journalctl@ceph.osd.6.smithi195.stdout:Sep 21 04:25:41 smithi195 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-6[57554]: 2024-09-21T04:25:41.614+0000 7f9870ab3640 -1 osd.6 165 heartbeat_check: no reply from 172.21.15.89:6814 osd.1 since back 2024-09-21T04:25:01.796546+0000 front 2024-09-21T04:25:01.796603+0000 (oldest deadline 2024-09-21T04:25:25.296494+0000) 2024-09-21T04:25:42.109 INFO:journalctl@ceph.osd.6.smithi195.stdout:Sep 21 04:25:41 smithi195 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-6[57554]: 2024-09-21T04:25:41.614+0000 7f9870ab3640 -1 osd.6 165 heartbeat_check: no reply from 172.21.15.89:6822 osd.2 since back 2024-09-21T04:25:07.597364+0000 front 2024-09-21T04:25:07.597396+0000 (oldest deadline 2024-09-21T04:25:30.497320+0000) 2024-09-21T04:25:42.109 INFO:journalctl@ceph.osd.6.smithi195.stdout:Sep 21 04:25:41 smithi195 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-6[57554]: 2024-09-21T04:25:41.614+0000 7f9870ab3640 -1 osd.6 165 heartbeat_check: no reply from 172.21.15.89:6830 osd.3 since back 2024-09-21T04:25:16.799127+0000 front 2024-09-21T04:25:16.799112+0000 (oldest deadline 2024-09-21T04:25:41.498989+0000) 2024-09-21T04:25:42.109 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 21 04:25:41 smithi195 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-7[59664]: 2024-09-21T04:25:41.737+0000 7f99a543c640 -1 osd.7 165 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2024-09-21T04:24:52.054834+0000 front 2024-09-21T04:24:52.054715+0000 (oldest deadline 2024-09-21T04:25:17.855130+0000) 2024-09-21T04:25:42.110 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 21 04:25:41 smithi195 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-7[59664]: 2024-09-21T04:25:41.737+0000 7f99a543c640 -1 osd.7 165 heartbeat_check: no reply from 172.21.15.89:6814 osd.1 since back 2024-09-21T04:25:01.356285+0000 front 2024-09-21T04:25:01.356363+0000 (oldest deadline 2024-09-21T04:25:26.556666+0000) 2024-09-21T04:25:42.110 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 21 04:25:41 smithi195 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-7[59664]: 2024-09-21T04:25:41.737+0000 7f99a543c640 -1 osd.7 165 heartbeat_check: no reply from 172.21.15.89:6822 osd.2 since back 2024-09-21T04:25:06.557206+0000 front 2024-09-21T04:25:06.557257+0000 (oldest deadline 2024-09-21T04:25:30.657081+0000) 2024-09-21T04:25:42.110 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 21 04:25:41 smithi195 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-7[59664]: 2024-09-21T04:25:41.737+0000 7f99a543c640 -1 osd.7 165 heartbeat_check: no reply from 172.21.15.89:6830 osd.3 since back 2024-09-21T04:25:15.358024+0000 front 2024-09-21T04:25:15.357954+0000 (oldest deadline 2024-09-21T04:25:39.958220+0000) 2024-09-21T04:25:42.457 INFO:journalctl@ceph.osd.6.smithi195.stdout:Sep 21 04:25:42 smithi195 podman[69502]: 2024-09-21 04:25:42.181533851 +0000 UTC m=+5.629408390 container died a41d0c0574ab7012ec0367c29c113be073f9c7c10e8132e59d41c848df04842a (image=quay.ceph.io/ceph-ci/ceph@sha256:2195e5f178be0a9390728e2a14088236d8fe6f82635f5c84418fe37a04457e00, name=ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-6, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, ceph=True, RELEASE=wip-guits-main-2024-09-20-2310-f36283e, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-21T04:25:42.724 INFO:journalctl@ceph.osd.6.smithi195.stdout:Sep 21 04:25:42 smithi195 podman[69502]: 2024-09-21 04:25:42.529113145 +0000 UTC m=+5.976987677 container cleanup a41d0c0574ab7012ec0367c29c113be073f9c7c10e8132e59d41c848df04842a (image=quay.ceph.io/ceph-ci/ceph@sha256:2195e5f178be0a9390728e2a14088236d8fe6f82635f5c84418fe37a04457e00, name=ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-6, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, RELEASE=wip-guits-main-2024-09-20-2310-f36283e, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23) 2024-09-21T04:25:42.724 INFO:journalctl@ceph.osd.6.smithi195.stdout:Sep 21 04:25:42 smithi195 bash[69502]: ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-6 2024-09-21T04:25:42.724 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 21 04:25:42 smithi195 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-7[59664]: 2024-09-21T04:25:42.691+0000 7f99a543c640 -1 osd.7 165 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2024-09-21T04:24:52.054834+0000 front 2024-09-21T04:24:52.054715+0000 (oldest deadline 2024-09-21T04:25:17.855130+0000) 2024-09-21T04:25:42.725 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 21 04:25:42 smithi195 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-7[59664]: 2024-09-21T04:25:42.691+0000 7f99a543c640 -1 osd.7 165 heartbeat_check: no reply from 172.21.15.89:6814 osd.1 since back 2024-09-21T04:25:01.356285+0000 front 2024-09-21T04:25:01.356363+0000 (oldest deadline 2024-09-21T04:25:26.556666+0000) 2024-09-21T04:25:42.725 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 21 04:25:42 smithi195 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-7[59664]: 2024-09-21T04:25:42.691+0000 7f99a543c640 -1 osd.7 165 heartbeat_check: no reply from 172.21.15.89:6822 osd.2 since back 2024-09-21T04:25:06.557206+0000 front 2024-09-21T04:25:06.557257+0000 (oldest deadline 2024-09-21T04:25:30.657081+0000) 2024-09-21T04:25:42.725 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 21 04:25:42 smithi195 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-7[59664]: 2024-09-21T04:25:42.691+0000 7f99a543c640 -1 osd.7 165 heartbeat_check: no reply from 172.21.15.89:6830 osd.3 since back 2024-09-21T04:25:15.358024+0000 front 2024-09-21T04:25:15.357954+0000 (oldest deadline 2024-09-21T04:25:39.958220+0000) 2024-09-21T04:25:43.109 INFO:journalctl@ceph.osd.6.smithi195.stdout:Sep 21 04:25:42 smithi195 podman[69517]: 2024-09-21 04:25:42.722281301 +0000 UTC m=+0.530618591 container remove a41d0c0574ab7012ec0367c29c113be073f9c7c10e8132e59d41c848df04842a (image=quay.ceph.io/ceph-ci/ceph@sha256:2195e5f178be0a9390728e2a14088236d8fe6f82635f5c84418fe37a04457e00, name=ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-6, ceph=True, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=wip-guits-main-2024-09-20-2310-f36283e, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731) 2024-09-21T04:25:43.469 INFO:journalctl@ceph.osd.6.smithi195.stdout:Sep 21 04:25:43 smithi195 podman[69607]: 2024-09-21 04:25:43.048413263 +0000 UTC m=+0.015865495 image pull 48ed0a2cf89d6d0371a2c6185a5869a747265f6abd5ac2a74f00cc32d5628721 quay.ceph.io/ceph-ci/ceph@sha256:2195e5f178be0a9390728e2a14088236d8fe6f82635f5c84418fe37a04457e00 2024-09-21T04:25:43.469 INFO:journalctl@ceph.osd.6.smithi195.stdout:Sep 21 04:25:43 smithi195 podman[69607]: 2024-09-21 04:25:43.156290823 +0000 UTC m=+0.123743059 container create fa7a1d917118752e64c835495c983890a68336d3ad08704a1b68caec713f4901 (image=quay.ceph.io/ceph-ci/ceph@sha256:2195e5f178be0a9390728e2a14088236d8fe6f82635f5c84418fe37a04457e00, name=ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-6-deactivate, ceph=True, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=wip-guits-main-2024-09-20-2310-f36283e, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_BRANCH=HEAD) 2024-09-21T04:25:43.469 INFO:journalctl@ceph.osd.6.smithi195.stdout:Sep 21 04:25:43 smithi195 podman[69607]: 2024-09-21 04:25:43.375340279 +0000 UTC m=+0.342792521 container init fa7a1d917118752e64c835495c983890a68336d3ad08704a1b68caec713f4901 (image=quay.ceph.io/ceph-ci/ceph@sha256:2195e5f178be0a9390728e2a14088236d8fe6f82635f5c84418fe37a04457e00, name=ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-6-deactivate, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , RELEASE=wip-guits-main-2024-09-20-2310-f36283e, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, ceph=True, CEPH_POINT_RELEASE=) 2024-09-21T04:25:43.469 INFO:journalctl@ceph.osd.6.smithi195.stdout:Sep 21 04:25:43 smithi195 podman[69607]: 2024-09-21 04:25:43.37864186 +0000 UTC m=+0.346094094 container start fa7a1d917118752e64c835495c983890a68336d3ad08704a1b68caec713f4901 (image=quay.ceph.io/ceph-ci/ceph@sha256:2195e5f178be0a9390728e2a14088236d8fe6f82635f5c84418fe37a04457e00, name=ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-6-deactivate, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=wip-guits-main-2024-09-20-2310-f36283e, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , ceph=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23) 2024-09-21T04:25:43.469 INFO:journalctl@ceph.osd.6.smithi195.stdout:Sep 21 04:25:43 smithi195 podman[69607]: 2024-09-21 04:25:43.467020536 +0000 UTC m=+0.434472792 container attach fa7a1d917118752e64c835495c983890a68336d3ad08704a1b68caec713f4901 (image=quay.ceph.io/ceph-ci/ceph@sha256:2195e5f178be0a9390728e2a14088236d8fe6f82635f5c84418fe37a04457e00, name=ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-6-deactivate, ceph=True, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, RELEASE=wip-guits-main-2024-09-20-2310-f36283e, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, GIT_CLEAN=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux ) 2024-09-21T04:25:43.859 INFO:journalctl@ceph.osd.6.smithi195.stdout:Sep 21 04:25:43 smithi195 podman[69607]: 2024-09-21 04:25:43.564589556 +0000 UTC m=+0.532041789 container died fa7a1d917118752e64c835495c983890a68336d3ad08704a1b68caec713f4901 (image=quay.ceph.io/ceph-ci/ceph@sha256:2195e5f178be0a9390728e2a14088236d8fe6f82635f5c84418fe37a04457e00, name=ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-6-deactivate, maintainer=Guillaume Abrioux , GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.build-date=20240731, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, RELEASE=wip-guits-main-2024-09-20-2310-f36283e, ceph=True, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-21T04:25:43.859 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 21 04:25:43 smithi195 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-7[59664]: 2024-09-21T04:25:43.647+0000 7f99a543c640 -1 osd.7 165 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2024-09-21T04:24:52.054834+0000 front 2024-09-21T04:24:52.054715+0000 (oldest deadline 2024-09-21T04:25:17.855130+0000) 2024-09-21T04:25:43.859 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 21 04:25:43 smithi195 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-7[59664]: 2024-09-21T04:25:43.647+0000 7f99a543c640 -1 osd.7 165 heartbeat_check: no reply from 172.21.15.89:6814 osd.1 since back 2024-09-21T04:25:01.356285+0000 front 2024-09-21T04:25:01.356363+0000 (oldest deadline 2024-09-21T04:25:26.556666+0000) 2024-09-21T04:25:43.859 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 21 04:25:43 smithi195 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-7[59664]: 2024-09-21T04:25:43.647+0000 7f99a543c640 -1 osd.7 165 heartbeat_check: no reply from 172.21.15.89:6822 osd.2 since back 2024-09-21T04:25:06.557206+0000 front 2024-09-21T04:25:06.557257+0000 (oldest deadline 2024-09-21T04:25:30.657081+0000) 2024-09-21T04:25:43.859 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 21 04:25:43 smithi195 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-7[59664]: 2024-09-21T04:25:43.647+0000 7f99a543c640 -1 osd.7 165 heartbeat_check: no reply from 172.21.15.89:6830 osd.3 since back 2024-09-21T04:25:15.358024+0000 front 2024-09-21T04:25:15.357954+0000 (oldest deadline 2024-09-21T04:25:39.958220+0000) 2024-09-21T04:25:44.163 DEBUG:teuthology.orchestra.run.smithi195:> sudo pkill -f 'journalctl -f -n 0 -u ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490@osd.6.service' 2024-09-21T04:25:44.201 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-21T04:25:44.201 INFO:tasks.cephadm.osd.6:Stopped osd.6 2024-09-21T04:25:44.201 INFO:tasks.cephadm.osd.7:Stopping osd.7... 2024-09-21T04:25:44.201 DEBUG:teuthology.orchestra.run.smithi195:> sudo systemctl stop ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490@osd.7 2024-09-21T04:25:44.542 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 21 04:25:44 smithi195 systemd[1]: Stopping Ceph osd.7 for 1b9b79fc-77ce-11ef-baf5-efdc52797490... 2024-09-21T04:25:44.543 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 21 04:25:44 smithi195 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-7[59664]: 2024-09-21T04:25:44.539+0000 7f99ace2f640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.7 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-21T04:25:44.543 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 21 04:25:44 smithi195 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-7[59664]: 2024-09-21T04:25:44.539+0000 7f99ace2f640 -1 osd.7 165 *** Got signal Terminated *** 2024-09-21T04:25:44.859 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 21 04:25:44 smithi195 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-7[59664]: 2024-09-21T04:25:44.539+0000 7f99ace2f640 -1 osd.7 165 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-21T04:25:44.859 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 21 04:25:44 smithi195 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-7[59664]: 2024-09-21T04:25:44.620+0000 7f99a543c640 -1 osd.7 165 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2024-09-21T04:24:52.054834+0000 front 2024-09-21T04:24:52.054715+0000 (oldest deadline 2024-09-21T04:25:17.855130+0000) 2024-09-21T04:25:44.859 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 21 04:25:44 smithi195 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-7[59664]: 2024-09-21T04:25:44.620+0000 7f99a543c640 -1 osd.7 165 heartbeat_check: no reply from 172.21.15.89:6814 osd.1 since back 2024-09-21T04:25:01.356285+0000 front 2024-09-21T04:25:01.356363+0000 (oldest deadline 2024-09-21T04:25:26.556666+0000) 2024-09-21T04:25:44.859 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 21 04:25:44 smithi195 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-7[59664]: 2024-09-21T04:25:44.620+0000 7f99a543c640 -1 osd.7 165 heartbeat_check: no reply from 172.21.15.89:6822 osd.2 since back 2024-09-21T04:25:06.557206+0000 front 2024-09-21T04:25:06.557257+0000 (oldest deadline 2024-09-21T04:25:30.657081+0000) 2024-09-21T04:25:44.859 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 21 04:25:44 smithi195 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-7[59664]: 2024-09-21T04:25:44.620+0000 7f99a543c640 -1 osd.7 165 heartbeat_check: no reply from 172.21.15.89:6830 osd.3 since back 2024-09-21T04:25:15.358024+0000 front 2024-09-21T04:25:15.357954+0000 (oldest deadline 2024-09-21T04:25:39.958220+0000) 2024-09-21T04:25:46.109 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 21 04:25:45 smithi195 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-7[59664]: 2024-09-21T04:25:45.658+0000 7f99a543c640 -1 osd.7 165 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2024-09-21T04:24:52.054834+0000 front 2024-09-21T04:24:52.054715+0000 (oldest deadline 2024-09-21T04:25:17.855130+0000) 2024-09-21T04:25:46.109 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 21 04:25:45 smithi195 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-7[59664]: 2024-09-21T04:25:45.658+0000 7f99a543c640 -1 osd.7 165 heartbeat_check: no reply from 172.21.15.89:6814 osd.1 since back 2024-09-21T04:25:01.356285+0000 front 2024-09-21T04:25:01.356363+0000 (oldest deadline 2024-09-21T04:25:26.556666+0000) 2024-09-21T04:25:46.109 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 21 04:25:45 smithi195 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-7[59664]: 2024-09-21T04:25:45.658+0000 7f99a543c640 -1 osd.7 165 heartbeat_check: no reply from 172.21.15.89:6822 osd.2 since back 2024-09-21T04:25:06.557206+0000 front 2024-09-21T04:25:06.557257+0000 (oldest deadline 2024-09-21T04:25:30.657081+0000) 2024-09-21T04:25:46.109 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 21 04:25:45 smithi195 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-7[59664]: 2024-09-21T04:25:45.658+0000 7f99a543c640 -1 osd.7 165 heartbeat_check: no reply from 172.21.15.89:6830 osd.3 since back 2024-09-21T04:25:15.358024+0000 front 2024-09-21T04:25:15.357954+0000 (oldest deadline 2024-09-21T04:25:39.958220+0000) 2024-09-21T04:25:47.109 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 21 04:25:46 smithi195 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-7[59664]: 2024-09-21T04:25:46.609+0000 7f99a543c640 -1 osd.7 165 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2024-09-21T04:24:52.054834+0000 front 2024-09-21T04:24:52.054715+0000 (oldest deadline 2024-09-21T04:25:17.855130+0000) 2024-09-21T04:25:47.109 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 21 04:25:46 smithi195 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-7[59664]: 2024-09-21T04:25:46.609+0000 7f99a543c640 -1 osd.7 165 heartbeat_check: no reply from 172.21.15.89:6814 osd.1 since back 2024-09-21T04:25:01.356285+0000 front 2024-09-21T04:25:01.356363+0000 (oldest deadline 2024-09-21T04:25:26.556666+0000) 2024-09-21T04:25:47.109 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 21 04:25:46 smithi195 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-7[59664]: 2024-09-21T04:25:46.609+0000 7f99a543c640 -1 osd.7 165 heartbeat_check: no reply from 172.21.15.89:6822 osd.2 since back 2024-09-21T04:25:06.557206+0000 front 2024-09-21T04:25:06.557257+0000 (oldest deadline 2024-09-21T04:25:30.657081+0000) 2024-09-21T04:25:47.109 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 21 04:25:46 smithi195 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-7[59664]: 2024-09-21T04:25:46.609+0000 7f99a543c640 -1 osd.7 165 heartbeat_check: no reply from 172.21.15.89:6830 osd.3 since back 2024-09-21T04:25:15.358024+0000 front 2024-09-21T04:25:15.357954+0000 (oldest deadline 2024-09-21T04:25:39.958220+0000) 2024-09-21T04:25:47.109 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 21 04:25:46 smithi195 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-7[59664]: 2024-09-21T04:25:46.609+0000 7f99a543c640 -1 osd.7 165 heartbeat_check: no reply from 172.21.15.195:6804 osd.4 since back 2024-09-21T04:25:24.559325+0000 front 2024-09-21T04:25:24.559305+0000 (oldest deadline 2024-09-21T04:25:46.259362+0000) 2024-09-21T04:25:48.010 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 21 04:25:47 smithi195 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-7[59664]: 2024-09-21T04:25:47.607+0000 7f99a543c640 -1 osd.7 165 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2024-09-21T04:24:52.054834+0000 front 2024-09-21T04:24:52.054715+0000 (oldest deadline 2024-09-21T04:25:17.855130+0000) 2024-09-21T04:25:48.011 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 21 04:25:47 smithi195 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-7[59664]: 2024-09-21T04:25:47.607+0000 7f99a543c640 -1 osd.7 165 heartbeat_check: no reply from 172.21.15.89:6814 osd.1 since back 2024-09-21T04:25:01.356285+0000 front 2024-09-21T04:25:01.356363+0000 (oldest deadline 2024-09-21T04:25:26.556666+0000) 2024-09-21T04:25:48.011 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 21 04:25:47 smithi195 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-7[59664]: 2024-09-21T04:25:47.607+0000 7f99a543c640 -1 osd.7 165 heartbeat_check: no reply from 172.21.15.89:6822 osd.2 since back 2024-09-21T04:25:06.557206+0000 front 2024-09-21T04:25:06.557257+0000 (oldest deadline 2024-09-21T04:25:30.657081+0000) 2024-09-21T04:25:48.011 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 21 04:25:47 smithi195 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-7[59664]: 2024-09-21T04:25:47.607+0000 7f99a543c640 -1 osd.7 165 heartbeat_check: no reply from 172.21.15.89:6830 osd.3 since back 2024-09-21T04:25:15.358024+0000 front 2024-09-21T04:25:15.357954+0000 (oldest deadline 2024-09-21T04:25:39.958220+0000) 2024-09-21T04:25:48.011 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 21 04:25:47 smithi195 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-7[59664]: 2024-09-21T04:25:47.607+0000 7f99a543c640 -1 osd.7 165 heartbeat_check: no reply from 172.21.15.195:6804 osd.4 since back 2024-09-21T04:25:24.559325+0000 front 2024-09-21T04:25:24.559305+0000 (oldest deadline 2024-09-21T04:25:46.259362+0000) 2024-09-21T04:25:48.861 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 21 04:25:48 smithi195 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-7[59664]: 2024-09-21T04:25:48.639+0000 7f99a543c640 -1 osd.7 165 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2024-09-21T04:24:52.054834+0000 front 2024-09-21T04:24:52.054715+0000 (oldest deadline 2024-09-21T04:25:17.855130+0000) 2024-09-21T04:25:48.861 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 21 04:25:48 smithi195 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-7[59664]: 2024-09-21T04:25:48.639+0000 7f99a543c640 -1 osd.7 165 heartbeat_check: no reply from 172.21.15.89:6814 osd.1 since back 2024-09-21T04:25:01.356285+0000 front 2024-09-21T04:25:01.356363+0000 (oldest deadline 2024-09-21T04:25:26.556666+0000) 2024-09-21T04:25:48.861 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 21 04:25:48 smithi195 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-7[59664]: 2024-09-21T04:25:48.639+0000 7f99a543c640 -1 osd.7 165 heartbeat_check: no reply from 172.21.15.89:6822 osd.2 since back 2024-09-21T04:25:06.557206+0000 front 2024-09-21T04:25:06.557257+0000 (oldest deadline 2024-09-21T04:25:30.657081+0000) 2024-09-21T04:25:48.861 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 21 04:25:48 smithi195 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-7[59664]: 2024-09-21T04:25:48.639+0000 7f99a543c640 -1 osd.7 165 heartbeat_check: no reply from 172.21.15.89:6830 osd.3 since back 2024-09-21T04:25:15.358024+0000 front 2024-09-21T04:25:15.357954+0000 (oldest deadline 2024-09-21T04:25:39.958220+0000) 2024-09-21T04:25:48.861 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 21 04:25:48 smithi195 ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-7[59664]: 2024-09-21T04:25:48.639+0000 7f99a543c640 -1 osd.7 165 heartbeat_check: no reply from 172.21.15.195:6804 osd.4 since back 2024-09-21T04:25:24.559325+0000 front 2024-09-21T04:25:24.559305+0000 (oldest deadline 2024-09-21T04:25:46.259362+0000) 2024-09-21T04:25:50.359 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 21 04:25:50 smithi195 podman[69711]: 2024-09-21 04:25:50.086796073 +0000 UTC m=+5.665209616 container died 66d3bf5f6c5fb57028aaee93e64eaeea1aee46159f007c42971b16fec7fe10e9 (image=quay.ceph.io/ceph-ci/ceph@sha256:2195e5f178be0a9390728e2a14088236d8fe6f82635f5c84418fe37a04457e00, name=ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-7, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_BRANCH=HEAD, RELEASE=wip-guits-main-2024-09-20-2310-f36283e, org.label-schema.vendor=CentOS, ceph=True, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux ) 2024-09-21T04:25:50.815 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 21 04:25:50 smithi195 podman[69711]: 2024-09-21 04:25:50.543495297 +0000 UTC m=+6.121908833 container cleanup 66d3bf5f6c5fb57028aaee93e64eaeea1aee46159f007c42971b16fec7fe10e9 (image=quay.ceph.io/ceph-ci/ceph@sha256:2195e5f178be0a9390728e2a14088236d8fe6f82635f5c84418fe37a04457e00, name=ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-7, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=wip-guits-main-2024-09-20-2310-f36283e, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_BRANCH=HEAD) 2024-09-21T04:25:50.815 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 21 04:25:50 smithi195 bash[69711]: ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-7 2024-09-21T04:25:51.109 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 21 04:25:50 smithi195 podman[69739]: 2024-09-21 04:25:50.812789979 +0000 UTC m=+0.720764112 container remove 66d3bf5f6c5fb57028aaee93e64eaeea1aee46159f007c42971b16fec7fe10e9 (image=quay.ceph.io/ceph-ci/ceph@sha256:2195e5f178be0a9390728e2a14088236d8fe6f82635f5c84418fe37a04457e00, name=ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-7, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, RELEASE=wip-guits-main-2024-09-20-2310-f36283e) 2024-09-21T04:25:51.468 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 21 04:25:51 smithi195 podman[69831]: 2024-09-21 04:25:51.06890213 +0000 UTC m=+0.015775851 image pull 48ed0a2cf89d6d0371a2c6185a5869a747265f6abd5ac2a74f00cc32d5628721 quay.ceph.io/ceph-ci/ceph@sha256:2195e5f178be0a9390728e2a14088236d8fe6f82635f5c84418fe37a04457e00 2024-09-21T04:25:51.468 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 21 04:25:51 smithi195 podman[69831]: 2024-09-21 04:25:51.180222958 +0000 UTC m=+0.127096716 container create 7b32bb187511f818cf4053d30a6d097f7717cf9fce63d90e66cbe27e7dba3f7a (image=quay.ceph.io/ceph-ci/ceph@sha256:2195e5f178be0a9390728e2a14088236d8fe6f82635f5c84418fe37a04457e00, name=ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-7-deactivate, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, RELEASE=wip-guits-main-2024-09-20-2310-f36283e, ceph=True, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True) 2024-09-21T04:25:51.468 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 21 04:25:51 smithi195 podman[69831]: 2024-09-21 04:25:51.39085165 +0000 UTC m=+0.337725367 container init 7b32bb187511f818cf4053d30a6d097f7717cf9fce63d90e66cbe27e7dba3f7a (image=quay.ceph.io/ceph-ci/ceph@sha256:2195e5f178be0a9390728e2a14088236d8fe6f82635f5c84418fe37a04457e00, name=ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-7-deactivate, org.label-schema.schema-version=1.0, GIT_CLEAN=True, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.license=GPLv2, RELEASE=wip-guits-main-2024-09-20-2310-f36283e, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS) 2024-09-21T04:25:51.468 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 21 04:25:51 smithi195 podman[69831]: 2024-09-21 04:25:51.394572714 +0000 UTC m=+0.341446427 container start 7b32bb187511f818cf4053d30a6d097f7717cf9fce63d90e66cbe27e7dba3f7a (image=quay.ceph.io/ceph-ci/ceph@sha256:2195e5f178be0a9390728e2a14088236d8fe6f82635f5c84418fe37a04457e00, name=ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-7-deactivate, io.buildah.version=1.37.2, GIT_CLEAN=True, RELEASE=wip-guits-main-2024-09-20-2310-f36283e, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731) 2024-09-21T04:25:51.468 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 21 04:25:51 smithi195 podman[69831]: 2024-09-21 04:25:51.465730971 +0000 UTC m=+0.412604700 container attach 7b32bb187511f818cf4053d30a6d097f7717cf9fce63d90e66cbe27e7dba3f7a (image=quay.ceph.io/ceph-ci/ceph@sha256:2195e5f178be0a9390728e2a14088236d8fe6f82635f5c84418fe37a04457e00, name=ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-7-deactivate, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.schema-version=1.0, RELEASE=wip-guits-main-2024-09-20-2310-f36283e, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-21T04:25:51.859 INFO:journalctl@ceph.osd.7.smithi195.stdout:Sep 21 04:25:51 smithi195 podman[69831]: 2024-09-21 04:25:51.584550832 +0000 UTC m=+0.531424548 container died 7b32bb187511f818cf4053d30a6d097f7717cf9fce63d90e66cbe27e7dba3f7a (image=quay.ceph.io/ceph-ci/ceph@sha256:2195e5f178be0a9390728e2a14088236d8fe6f82635f5c84418fe37a04457e00, name=ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490-osd-7-deactivate, org.label-schema.license=GPLv2, GIT_CLEAN=True, CEPH_POINT_RELEASE=, ceph=True, io.buildah.version=1.37.2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, RELEASE=wip-guits-main-2024-09-20-2310-f36283e, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-21T04:25:52.140 DEBUG:teuthology.orchestra.run.smithi195:> sudo pkill -f 'journalctl -f -n 0 -u ceph-1b9b79fc-77ce-11ef-baf5-efdc52797490@osd.7.service' 2024-09-21T04:25:52.178 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-21T04:25:52.178 INFO:tasks.cephadm.osd.7:Stopped osd.7 2024-09-21T04:25:52.178 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 1b9b79fc-77ce-11ef-baf5-efdc52797490 --force --keep-logs 2024-09-21T04:28:48.654 DEBUG:teuthology.orchestra.run.smithi195:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 1b9b79fc-77ce-11ef-baf5-efdc52797490 --force --keep-logs 2024-09-21T04:29:10.022 DEBUG:teuthology.orchestra.run.smithi089:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-09-21T04:29:10.057 DEBUG:teuthology.orchestra.run.smithi195:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-09-21T04:29:10.085 INFO:tasks.cephadm:Archiving crash dumps... 2024-09-21T04:29:10.087 DEBUG:teuthology.misc:Transferring archived files from smithi089:/var/lib/ceph/1b9b79fc-77ce-11ef-baf5-efdc52797490/crash to /home/teuthworker/archive/gabrioux-2024-09-21_03:44:43-orch:cephadm-wip-guits-main-2024-09-20-2310-distro-default-smithi/7913679/remote/smithi089/crash 2024-09-21T04:29:10.087 DEBUG:teuthology.orchestra.run.smithi089:> sudo tar c -f - -C /var/lib/ceph/1b9b79fc-77ce-11ef-baf5-efdc52797490/crash -- . 2024-09-21T04:29:10.145 INFO:teuthology.orchestra.run.smithi089.stderr:tar: /var/lib/ceph/1b9b79fc-77ce-11ef-baf5-efdc52797490/crash: Cannot open: No such file or directory 2024-09-21T04:29:10.145 INFO:teuthology.orchestra.run.smithi089.stderr:tar: Error is not recoverable: exiting now 2024-09-21T04:29:10.147 DEBUG:teuthology.misc:Transferring archived files from smithi195:/var/lib/ceph/1b9b79fc-77ce-11ef-baf5-efdc52797490/crash to /home/teuthworker/archive/gabrioux-2024-09-21_03:44:43-orch:cephadm-wip-guits-main-2024-09-20-2310-distro-default-smithi/7913679/remote/smithi195/crash 2024-09-21T04:29:10.150 DEBUG:teuthology.orchestra.run.smithi195:> sudo tar c -f - -C /var/lib/ceph/1b9b79fc-77ce-11ef-baf5-efdc52797490/crash -- . 2024-09-21T04:29:10.184 INFO:teuthology.orchestra.run.smithi195.stderr:tar: /var/lib/ceph/1b9b79fc-77ce-11ef-baf5-efdc52797490/crash: Cannot open: No such file or directory 2024-09-21T04:29:10.185 INFO:teuthology.orchestra.run.smithi195.stderr:tar: Error is not recoverable: exiting now 2024-09-21T04:29:10.186 INFO:tasks.cephadm:Checking cluster log for badness... 2024-09-21T04:29:10.186 DEBUG:teuthology.orchestra.run.smithi089:> sudo egrep '\[ERR\]|\[WRN\]|\[SEC\]' /var/log/ceph/1b9b79fc-77ce-11ef-baf5-efdc52797490/ceph.log | egrep CEPHADM_ | egrep -v '\(MDS_ALL_DOWN\)' | egrep -v '\(MDS_UP_LESS_THAN_MAX\)' | egrep -v CEPHADM_REFRESH_FAILED | head -n 1 2024-09-21T04:29:10.217 INFO:tasks.cephadm:Compressing logs... 2024-09-21T04:29:10.217 DEBUG:teuthology.orchestra.run.smithi089:> time sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs --max-args=1 --max-procs=0 --verbose -0 --no-run-if-empty -- gzip -5 --verbose -- 2024-09-21T04:29:10.260 DEBUG:teuthology.orchestra.run.smithi195:> time sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs --max-args=1 --max-procs=0 --verbose -0 --no-run-if-empty -- gzip -5 --verbose -- 2024-09-21T04:29:10.283 INFO:teuthology.orchestra.run.smithi089.stderr:find: ‘/var/log/rbd-target-api’gzip -5 --verbose --: No such file or directory /var/log/ceph/1b9b79fc-77ce-11ef-baf5-efdc52797490/ceph-osd.0.log 2024-09-21T04:29:10.283 INFO:teuthology.orchestra.run.smithi089.stderr: 2024-09-21T04:29:10.283 INFO:teuthology.orchestra.run.smithi089.stderr:gzip -5 --verbose -- /var/log/ceph/1b9b79fc-77ce-11ef-baf5-efdc52797490/ceph-mds.foofs.smithi089.osufws.log 2024-09-21T04:29:10.283 INFO:teuthology.orchestra.run.smithi089.stderr:gzip -5 --verbose -- /var/log/ceph/1b9b79fc-77ce-11ef-baf5-efdc52797490/ceph-osd.1.log 2024-09-21T04:29:10.288 INFO:teuthology.orchestra.run.smithi089.stderr:/var/log/ceph/1b9b79fc-77ce-11ef-baf5-efdc52797490/ceph-osd.0.log: /var/log/ceph/1b9b79fc-77ce-11ef-baf5-efdc52797490/ceph-mds.foofs.smithi089.osufws.log: gzip -5 --verbose -- /var/log/ceph/1b9b79fc-77ce-11ef-baf5-efdc52797490/ceph-volume.log 2024-09-21T04:29:10.288 INFO:teuthology.orchestra.run.smithi089.stderr:/var/log/ceph/1b9b79fc-77ce-11ef-baf5-efdc52797490/ceph-osd.1.log: gzip -5 --verbose -- /var/log/ceph/1b9b79fc-77ce-11ef-baf5-efdc52797490/ceph.audit.log 2024-09-21T04:29:10.289 INFO:teuthology.orchestra.run.smithi089.stderr: 89.2% -- replaced with /var/log/ceph/1b9b79fc-77ce-11ef-baf5-efdc52797490/ceph-mds.foofs.smithi089.osufws.log.gz 2024-09-21T04:29:10.289 INFO:teuthology.orchestra.run.smithi089.stderr:/var/log/ceph/1b9b79fc-77ce-11ef-baf5-efdc52797490/ceph-volume.log: gzip -5 --verbose -- /var/log/ceph/1b9b79fc-77ce-11ef-baf5-efdc52797490/ceph-mon.smithi089.log 2024-09-21T04:29:10.289 INFO:teuthology.orchestra.run.smithi089.stderr:gzip -5 --verbose -- /var/log/ceph/1b9b79fc-77ce-11ef-baf5-efdc52797490/ceph.cephadm.log 2024-09-21T04:29:10.289 INFO:teuthology.orchestra.run.smithi089.stderr:/var/log/ceph/1b9b79fc-77ce-11ef-baf5-efdc52797490/ceph.audit.log: /var/log/ceph/1b9b79fc-77ce-11ef-baf5-efdc52797490/ceph-mon.smithi089.log: gzip -5 --verbose -- /var/log/ceph/1b9b79fc-77ce-11ef-baf5-efdc52797490/ceph-mgr.smithi089.srlyxf.log 2024-09-21T04:29:10.289 INFO:teuthology.orchestra.run.smithi089.stderr:/var/log/ceph/1b9b79fc-77ce-11ef-baf5-efdc52797490/ceph.cephadm.log: gzip -5 --verbose -- /var/log/ceph/1b9b79fc-77ce-11ef-baf5-efdc52797490/ceph-osd.2.log 2024-09-21T04:29:10.289 INFO:teuthology.orchestra.run.smithi089.stderr:/var/log/ceph/1b9b79fc-77ce-11ef-baf5-efdc52797490/ceph-mgr.smithi089.srlyxf.log: gzip -5 --verbose -- /var/log/ceph/1b9b79fc-77ce-11ef-baf5-efdc52797490/ceph.log 2024-09-21T04:29:10.289 INFO:teuthology.orchestra.run.smithi089.stderr: 85.7% -- replaced with /var/log/ceph/1b9b79fc-77ce-11ef-baf5-efdc52797490/ceph.cephadm.log.gz 2024-09-21T04:29:10.289 INFO:teuthology.orchestra.run.smithi089.stderr:/var/log/ceph/1b9b79fc-77ce-11ef-baf5-efdc52797490/ceph-osd.2.log: gzip -5 --verbose -- /var/log/ceph/1b9b79fc-77ce-11ef-baf5-efdc52797490/ceph-osd.3.log 2024-09-21T04:29:10.289 INFO:teuthology.orchestra.run.smithi195.stderr:find: ‘/var/log/rbd-target-api’gzip -5 --verbose -- /var/log/ceph/1b9b79fc-77ce-11ef-baf5-efdc52797490/ceph-osd.7.log 2024-09-21T04:29:10.289 INFO:teuthology.orchestra.run.smithi195.stderr:: No such file or directory 2024-09-21T04:29:10.289 INFO:teuthology.orchestra.run.smithi195.stderr:gzip -5 --verbose -- /var/log/ceph/1b9b79fc-77ce-11ef-baf5-efdc52797490/ceph-mon.smithi195.log 2024-09-21T04:29:10.289 INFO:teuthology.orchestra.run.smithi195.stderr:/var/log/ceph/1b9b79fc-77ce-11ef-baf5-efdc52797490/ceph-osd.7.log: gzip -5 --verbose -- /var/log/ceph/1b9b79fc-77ce-11ef-baf5-efdc52797490/ceph-volume.log 2024-09-21T04:29:10.289 INFO:teuthology.orchestra.run.smithi195.stderr:/var/log/ceph/1b9b79fc-77ce-11ef-baf5-efdc52797490/ceph-mon.smithi195.log: gzip -5 --verbose -- /var/log/ceph/1b9b79fc-77ce-11ef-baf5-efdc52797490/ceph-osd.4.log 2024-09-21T04:29:10.290 INFO:teuthology.orchestra.run.smithi195.stderr:/var/log/ceph/1b9b79fc-77ce-11ef-baf5-efdc52797490/ceph-volume.log: gzip -5 --verbose -- /var/log/ceph/1b9b79fc-77ce-11ef-baf5-efdc52797490/ceph.audit.log 2024-09-21T04:29:10.290 INFO:teuthology.orchestra.run.smithi195.stderr:gzip -5 --verbose -- /var/log/ceph/1b9b79fc-77ce-11ef-baf5-efdc52797490/ceph.cephadm.log 2024-09-21T04:29:10.290 INFO:teuthology.orchestra.run.smithi195.stderr:/var/log/ceph/1b9b79fc-77ce-11ef-baf5-efdc52797490/ceph-osd.4.log: /var/log/ceph/1b9b79fc-77ce-11ef-baf5-efdc52797490/ceph.audit.log: gzip -5 --verbose -- /var/log/ceph/1b9b79fc-77ce-11ef-baf5-efdc52797490/ceph-osd.6.log 2024-09-21T04:29:10.290 INFO:teuthology.orchestra.run.smithi195.stderr:/var/log/ceph/1b9b79fc-77ce-11ef-baf5-efdc52797490/ceph.cephadm.log: gzip -5 --verbose -- /var/log/ceph/1b9b79fc-77ce-11ef-baf5-efdc52797490/ceph-osd.5.log 2024-09-21T04:29:10.290 INFO:teuthology.orchestra.run.smithi195.stderr:/var/log/ceph/1b9b79fc-77ce-11ef-baf5-efdc52797490/ceph-osd.6.log: gzip -5 --verbose -- /var/log/ceph/1b9b79fc-77ce-11ef-baf5-efdc52797490/ceph.log 2024-09-21T04:29:10.290 INFO:teuthology.orchestra.run.smithi195.stderr: 85.6% -- replaced with /var/log/ceph/1b9b79fc-77ce-11ef-baf5-efdc52797490/ceph.cephadm.log.gz 2024-09-21T04:29:10.290 INFO:teuthology.orchestra.run.smithi195.stderr:/var/log/ceph/1b9b79fc-77ce-11ef-baf5-efdc52797490/ceph-osd.5.log: 91.6% -- replaced with /var/log/ceph/1b9b79fc-77ce-11ef-baf5-efdc52797490/ceph.audit.log.gz 2024-09-21T04:29:10.291 INFO:teuthology.orchestra.run.smithi089.stderr:/var/log/ceph/1b9b79fc-77ce-11ef-baf5-efdc52797490/ceph.log: 91.5% -- replaced with /var/log/ceph/1b9b79fc-77ce-11ef-baf5-efdc52797490/ceph.audit.log.gz 2024-09-21T04:29:10.291 INFO:teuthology.orchestra.run.smithi089.stderr: 87.0% -- replaced with /var/log/ceph/1b9b79fc-77ce-11ef-baf5-efdc52797490/ceph.log.gz 2024-09-21T04:29:10.291 INFO:teuthology.orchestra.run.smithi089.stderr:gzip -5 --verbose -- /var/log/ceph/cephadm.log 2024-09-21T04:29:10.295 INFO:teuthology.orchestra.run.smithi195.stderr:gzip -5 --verbose -- /var/log/ceph/1b9b79fc-77ce-11ef-baf5-efdc52797490/ceph-mds.foofs.smithi195.hijxre.log 2024-09-21T04:29:10.295 INFO:teuthology.orchestra.run.smithi195.stderr:/var/log/ceph/1b9b79fc-77ce-11ef-baf5-efdc52797490/ceph.log: gzip -5 --verbose -- /var/log/ceph/1b9b79fc-77ce-11ef-baf5-efdc52797490/ceph-mgr.smithi195.cgotzm.log 2024-09-21T04:29:10.296 INFO:teuthology.orchestra.run.smithi195.stderr:/var/log/ceph/1b9b79fc-77ce-11ef-baf5-efdc52797490/ceph-mds.foofs.smithi195.hijxre.log: 88.5% -- replaced with /var/log/ceph/1b9b79fc-77ce-11ef-baf5-efdc52797490/ceph-mds.foofs.smithi195.hijxre.log.gz 2024-09-21T04:29:10.297 INFO:teuthology.orchestra.run.smithi195.stderr:gzip -5 --verbose -- /var/log/ceph/cephadm.log 2024-09-21T04:29:10.298 INFO:teuthology.orchestra.run.smithi195.stderr:/var/log/ceph/1b9b79fc-77ce-11ef-baf5-efdc52797490/ceph-mgr.smithi195.cgotzm.log: 87.0% -- replaced with /var/log/ceph/1b9b79fc-77ce-11ef-baf5-efdc52797490/ceph.log.gz 2024-09-21T04:29:10.303 INFO:teuthology.orchestra.run.smithi089.stderr:/var/log/ceph/1b9b79fc-77ce-11ef-baf5-efdc52797490/ceph-osd.3.log: /var/log/ceph/cephadm.log: 91.5% -- replaced with /var/log/ceph/1b9b79fc-77ce-11ef-baf5-efdc52797490/ceph-volume.log.gz 2024-09-21T04:29:10.303 INFO:teuthology.orchestra.run.smithi195.stderr:/var/log/ceph/cephadm.log: 91.6% -- replaced with /var/log/ceph/1b9b79fc-77ce-11ef-baf5-efdc52797490/ceph-volume.log.gz 2024-09-21T04:29:10.306 INFO:teuthology.orchestra.run.smithi195.stderr: 90.1% -- replaced with /var/log/ceph/cephadm.log.gz 2024-09-21T04:29:10.331 INFO:teuthology.orchestra.run.smithi089.stderr: 91.8% -- replaced with /var/log/ceph/cephadm.log.gz 2024-09-21T04:29:10.367 INFO:teuthology.orchestra.run.smithi195.stderr: 89.7% -- replaced with /var/log/ceph/1b9b79fc-77ce-11ef-baf5-efdc52797490/ceph-mgr.smithi195.cgotzm.log.gz 2024-09-21T04:29:10.564 INFO:teuthology.orchestra.run.smithi089.stderr: 88.6% -- replaced with /var/log/ceph/1b9b79fc-77ce-11ef-baf5-efdc52797490/ceph-mgr.smithi089.srlyxf.log.gz 2024-09-21T04:29:10.612 INFO:teuthology.orchestra.run.smithi195.stderr: 92.6% -- replaced with /var/log/ceph/1b9b79fc-77ce-11ef-baf5-efdc52797490/ceph-mon.smithi195.log.gz 2024-09-21T04:29:11.635 INFO:teuthology.orchestra.run.smithi089.stderr: 91.9% -- replaced with /var/log/ceph/1b9b79fc-77ce-11ef-baf5-efdc52797490/ceph-mon.smithi089.log.gz 2024-09-21T04:29:24.871 INFO:teuthology.orchestra.run.smithi089.stderr: 94.4% -- replaced with /var/log/ceph/1b9b79fc-77ce-11ef-baf5-efdc52797490/ceph-osd.2.log.gz 2024-09-21T04:29:25.176 INFO:teuthology.orchestra.run.smithi195.stderr: 94.3% -- replaced with /var/log/ceph/1b9b79fc-77ce-11ef-baf5-efdc52797490/ceph-osd.6.log.gz 2024-09-21T04:29:25.232 INFO:teuthology.orchestra.run.smithi195.stderr: 93.9% -- replaced with /var/log/ceph/1b9b79fc-77ce-11ef-baf5-efdc52797490/ceph-osd.4.log.gz 2024-09-21T04:29:26.913 INFO:teuthology.orchestra.run.smithi089.stderr: 93.9% -- replaced with /var/log/ceph/1b9b79fc-77ce-11ef-baf5-efdc52797490/ceph-osd.0.log.gz 2024-09-21T04:29:26.961 INFO:teuthology.orchestra.run.smithi089.stderr: 94.1% -- replaced with /var/log/ceph/1b9b79fc-77ce-11ef-baf5-efdc52797490/ceph-osd.3.log.gz 2024-09-21T04:29:28.944 INFO:teuthology.orchestra.run.smithi089.stderr: 94.2% -- replaced with /var/log/ceph/1b9b79fc-77ce-11ef-baf5-efdc52797490/ceph-osd.1.log.gz 2024-09-21T04:29:28.945 INFO:teuthology.orchestra.run.smithi089.stderr: 2024-09-21T04:29:28.945 INFO:teuthology.orchestra.run.smithi089.stderr:real 0m18.673s 2024-09-21T04:29:28.945 INFO:teuthology.orchestra.run.smithi089.stderr:user 1m5.470s 2024-09-21T04:29:28.945 INFO:teuthology.orchestra.run.smithi089.stderr:sys 0m2.663s 2024-09-21T04:29:29.776 INFO:teuthology.orchestra.run.smithi195.stderr: 94.2% -- replaced with /var/log/ceph/1b9b79fc-77ce-11ef-baf5-efdc52797490/ceph-osd.5.log.gz 2024-09-21T04:29:31.501 INFO:teuthology.orchestra.run.smithi195.stderr: 94.1% -- replaced with /var/log/ceph/1b9b79fc-77ce-11ef-baf5-efdc52797490/ceph-osd.7.log.gz 2024-09-21T04:29:31.504 INFO:teuthology.orchestra.run.smithi195.stderr: 2024-09-21T04:29:31.504 INFO:teuthology.orchestra.run.smithi195.stderr:real 0m21.230s 2024-09-21T04:29:31.504 INFO:teuthology.orchestra.run.smithi195.stderr:user 1m8.043s 2024-09-21T04:29:31.504 INFO:teuthology.orchestra.run.smithi195.stderr:sys 0m2.724s 2024-09-21T04:29:31.504 INFO:tasks.cephadm:Archiving logs... 2024-09-21T04:29:31.505 DEBUG:teuthology.misc:Transferring archived files from smithi089:/var/log/ceph to /home/teuthworker/archive/gabrioux-2024-09-21_03:44:43-orch:cephadm-wip-guits-main-2024-09-20-2310-distro-default-smithi/7913679/remote/smithi089/log 2024-09-21T04:29:31.506 DEBUG:teuthology.orchestra.run.smithi089:> sudo tar c -f - -C /var/log/ceph -- . 2024-09-21T04:29:40.339 DEBUG:teuthology.misc:Transferring archived files from smithi195:/var/log/ceph to /home/teuthworker/archive/gabrioux-2024-09-21_03:44:43-orch:cephadm-wip-guits-main-2024-09-20-2310-distro-default-smithi/7913679/remote/smithi195/log 2024-09-21T04:29:40.340 DEBUG:teuthology.orchestra.run.smithi195:> sudo tar c -f - -C /var/log/ceph -- . 2024-09-21T04:29:49.568 INFO:tasks.cephadm:Removing cluster... 2024-09-21T04:29:49.568 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 1b9b79fc-77ce-11ef-baf5-efdc52797490 --force 2024-09-21T04:29:49.932 DEBUG:teuthology.orchestra.run.smithi195:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 1b9b79fc-77ce-11ef-baf5-efdc52797490 --force 2024-09-21T04:29:50.304 INFO:tasks.cephadm:Removing cephadm ... 2024-09-21T04:29:50.304 DEBUG:teuthology.orchestra.run.smithi089:> rm -rf /home/ubuntu/cephtest/cephadm 2024-09-21T04:29:50.320 DEBUG:teuthology.orchestra.run.smithi195:> rm -rf /home/ubuntu/cephtest/cephadm 2024-09-21T04:29:50.336 INFO:tasks.cephadm:Teardown complete 2024-09-21T04:29:50.336 DEBUG:teuthology.run_tasks:Unwinding manager clock 2024-09-21T04:29:50.345 INFO:teuthology.task.clock:Checking final clock skew... 2024-09-21T04:29:50.345 DEBUG:teuthology.orchestra.run.smithi089:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-09-21T04:29:50.363 DEBUG:teuthology.orchestra.run.smithi195:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-09-21T04:29:50.375 INFO:teuthology.orchestra.run.smithi089.stderr:bash: line 1: ntpq: command not found 2024-09-21T04:29:50.379 INFO:teuthology.orchestra.run.smithi089.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-09-21T04:29:50.379 INFO:teuthology.orchestra.run.smithi089.stdout:=============================================================================== 2024-09-21T04:29:50.379 INFO:teuthology.orchestra.run.smithi089.stdout:^* hv01.front.sepia.ceph.com 3 6 377 42 -460us[ -550us] +/- 35ms 2024-09-21T04:29:50.379 INFO:teuthology.orchestra.run.smithi089.stdout:^+ hv02.front.sepia.ceph.com 3 6 377 44 +1658us[+1567us] +/- 76ms 2024-09-21T04:29:50.379 INFO:teuthology.orchestra.run.smithi089.stdout:^+ hv03.front.sepia.ceph.com 3 6 377 45 -546us[ -637us] +/- 66ms 2024-09-21T04:29:50.379 INFO:teuthology.orchestra.run.smithi089.stdout:^? hv04.front.sepia.ceph.com 0 9 0 - +0ns[ +0ns] +/- 0ns 2024-09-21T04:29:50.390 INFO:teuthology.orchestra.run.smithi195.stderr:bash: line 1: ntpq: command not found 2024-09-21T04:29:50.394 INFO:teuthology.orchestra.run.smithi195.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-09-21T04:29:50.394 INFO:teuthology.orchestra.run.smithi195.stdout:=============================================================================== 2024-09-21T04:29:50.394 INFO:teuthology.orchestra.run.smithi195.stdout:^* hv01.front.sepia.ceph.com 3 6 377 42 -466us[ -541us] +/- 35ms 2024-09-21T04:29:50.395 INFO:teuthology.orchestra.run.smithi195.stdout:^+ hv02.front.sepia.ceph.com 3 6 377 45 +1660us[+1584us] +/- 76ms 2024-09-21T04:29:50.395 INFO:teuthology.orchestra.run.smithi195.stdout:^+ hv03.front.sepia.ceph.com 3 6 377 46 -534us[ -610us] +/- 66ms 2024-09-21T04:29:50.395 INFO:teuthology.orchestra.run.smithi195.stdout:^? hv04.front.sepia.ceph.com 0 9 0 - +0ns[ +0ns] +/- 0ns 2024-09-21T04:29:50.396 DEBUG:teuthology.run_tasks:Unwinding manager ansible.cephlab 2024-09-21T04:29:50.405 INFO:teuthology.task.ansible:Skipping ansible cleanup... 2024-09-21T04:29:50.406 DEBUG:teuthology.run_tasks:Unwinding manager selinux 2024-09-21T04:29:50.440 DEBUG:teuthology.orchestra.run.smithi089:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2024-09-21T04:29:50.515 DEBUG:teuthology.orchestra.run.smithi195:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2024-09-21T04:29:50.586 DEBUG:teuthology.orchestra.run.smithi089:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e context=system_u:system_r:getty_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 2024-09-21T04:29:50.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T04:29:50.614 DEBUG:teuthology.orchestra.run.smithi195:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e context=system_u:system_r:getty_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 2024-09-21T04:29:50.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T04:29:50.642 DEBUG:teuthology.run_tasks:Unwinding manager pcp 2024-09-21T04:29:50.656 DEBUG:teuthology.run_tasks:Unwinding manager internal.timer 2024-09-21T04:29:50.665 INFO:teuthology.task.internal:Duration was 1957.388919 seconds 2024-09-21T04:29:50.666 DEBUG:teuthology.run_tasks:Unwinding manager internal.syslog 2024-09-21T04:29:50.674 INFO:teuthology.task.internal.syslog:Shutting down syslog monitoring... 2024-09-21T04:29:50.675 DEBUG:teuthology.orchestra.run.smithi089:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2024-09-21T04:29:50.677 DEBUG:teuthology.orchestra.run.smithi195:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2024-09-21T04:29:50.713 INFO:teuthology.orchestra.run.smithi089.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-09-21T04:29:50.721 INFO:teuthology.orchestra.run.smithi195.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-09-21T04:29:50.896 INFO:teuthology.task.internal.syslog:Checking logs for errors... 2024-09-21T04:29:50.896 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi089.front.sepia.ceph.com 2024-09-21T04:29:50.896 DEBUG:teuthology.orchestra.run.smithi089:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2024-09-21T04:29:50.917 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi195.front.sepia.ceph.com 2024-09-21T04:29:50.917 DEBUG:teuthology.orchestra.run.smithi195:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2024-09-21T04:29:50.959 INFO:teuthology.task.internal.syslog:Compressing syslogs... 2024-09-21T04:29:50.959 DEBUG:teuthology.orchestra.run.smithi089:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-09-21T04:29:50.961 DEBUG:teuthology.orchestra.run.smithi195:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-09-21T04:29:51.026 INFO:teuthology.task.internal.syslog:Gathering journactl -b0... 2024-09-21T04:29:51.026 DEBUG:teuthology.orchestra.run.smithi089:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2024-09-21T04:29:51.028 DEBUG:teuthology.orchestra.run.smithi195:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2024-09-21T04:30:04.876 DEBUG:teuthology.run_tasks:Unwinding manager internal.sudo 2024-09-21T04:30:04.886 INFO:teuthology.task.internal:Restoring /etc/sudoers... 2024-09-21T04:30:04.887 DEBUG:teuthology.orchestra.run.smithi089:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2024-09-21T04:30:04.912 DEBUG:teuthology.orchestra.run.smithi195:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2024-09-21T04:30:04.945 DEBUG:teuthology.run_tasks:Unwinding manager internal.coredump 2024-09-21T04:30:04.954 DEBUG:teuthology.orchestra.run.smithi089:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2024-09-21T04:30:04.957 DEBUG:teuthology.orchestra.run.smithi195:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2024-09-21T04:30:04.978 INFO:teuthology.orchestra.run.smithi089.stdout:kernel.core_pattern = core 2024-09-21T04:30:05.013 INFO:teuthology.orchestra.run.smithi195.stdout:kernel.core_pattern = core 2024-09-21T04:30:05.047 DEBUG:teuthology.orchestra.run.smithi089:> test -e /home/ubuntu/cephtest/archive/coredump 2024-09-21T04:30:05.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T04:30:05.062 DEBUG:teuthology.orchestra.run.smithi195:> test -e /home/ubuntu/cephtest/archive/coredump 2024-09-21T04:30:05.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T04:30:05.104 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive 2024-09-21T04:30:05.117 INFO:teuthology.task.internal:Transferring archived files... 2024-09-21T04:30:05.117 DEBUG:teuthology.misc:Transferring archived files from smithi089:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/gabrioux-2024-09-21_03:44:43-orch:cephadm-wip-guits-main-2024-09-20-2310-distro-default-smithi/7913679/remote/smithi089 2024-09-21T04:30:05.118 DEBUG:teuthology.orchestra.run.smithi089:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2024-09-21T04:30:05.178 DEBUG:teuthology.misc:Transferring archived files from smithi195:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/gabrioux-2024-09-21_03:44:43-orch:cephadm-wip-guits-main-2024-09-20-2310-distro-default-smithi/7913679/remote/smithi195 2024-09-21T04:30:05.178 DEBUG:teuthology.orchestra.run.smithi195:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2024-09-21T04:30:05.219 INFO:teuthology.task.internal:Removing archive directory... 2024-09-21T04:30:05.219 DEBUG:teuthology.orchestra.run.smithi089:> rm -rf -- /home/ubuntu/cephtest/archive 2024-09-21T04:30:05.221 DEBUG:teuthology.orchestra.run.smithi195:> rm -rf -- /home/ubuntu/cephtest/archive 2024-09-21T04:30:05.269 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive_upload 2024-09-21T04:30:05.281 INFO:teuthology.task.internal:Not uploading archives. 2024-09-21T04:30:05.281 DEBUG:teuthology.run_tasks:Unwinding manager internal.base 2024-09-21T04:30:05.299 INFO:teuthology.task.internal:Tidying up after the test... 2024-09-21T04:30:05.299 DEBUG:teuthology.orchestra.run.smithi089:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2024-09-21T04:30:05.301 DEBUG:teuthology.orchestra.run.smithi195:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2024-09-21T04:30:05.314 INFO:teuthology.orchestra.run.smithi089.stdout: 266063 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Sep 21 04:30 /home/ubuntu/cephtest 2024-09-21T04:30:05.326 INFO:teuthology.orchestra.run.smithi195.stdout: 265324 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Sep 21 04:30 /home/ubuntu/cephtest 2024-09-21T04:30:05.327 DEBUG:teuthology.run_tasks:Unwinding manager kernel 2024-09-21T04:30:05.339 DEBUG:teuthology.run_tasks:Unwinding manager console_log 2024-09-21T04:30:05.371 INFO:teuthology.run:Summary data: description: orch:cephadm/mgr-nfs-upgrade/{0-centos_9.stream 1-bootstrap/17.2.0 1-start 2-nfs 3-upgrade-with-workload 4-final} duration: 1957.388918876648 owner: scheduled_gabrioux@teuthology success: true 2024-09-21T04:30:05.371 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2024-09-21T04:30:05.453 INFO:teuthology.run:pass