2024-09-16T07:57:45.337 INFO:root:teuthology version: 1.2.2.dev2+g3752d38 2024-09-16T07:57:45.356 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2024-09-16T07:57:45.439 INFO:teuthology.run:Config: archive_path: /home/teuthworker/archive/gabrioux-2024-09-16_07:44:22-orch:cephadm-wip-guits-main-2024-09-13-1248-distro-default-smithi/7906916 branch: wip-guits-main-2024-09-13-1248 description: orch:cephadm/mgr-nfs-upgrade/{0-centos_9.stream 1-bootstrap/17.2.0 1-start 2-nfs 3-upgrade-with-workload 4-final} email: gabrioux@ibm.com first_in_suite: false job_id: '7906916' kernel: kdb: 1 sha1: distro last_in_suite: false machine_type: smithi name: gabrioux-2024-09-16_07:44:22-orch:cephadm-wip-guits-main-2024-09-13-1248-distro-default-smithi no_nested_subset: false openstack: - volumes: count: 4 size: 10 os_type: centos os_version: 9.stream overrides: admin_socket: branch: wip-guits-main-2024-09-13-1248 ceph: conf: mgr: debug mgr: 20 debug ms: 1 mon: debug mon: 20 debug ms: 1 debug paxos: 20 osd: debug ms: 1 debug osd: 20 osd shutdown pgref assert: true flavor: default log-ignorelist: - \(MDS_ALL_DOWN\) - \(MDS_UP_LESS_THAN_MAX\) - CEPHADM_REFRESH_FAILED log-only-match: - CEPHADM_ sha1: 8293d73f8690540e843a81caec373f9cc29cf705 ceph-deploy: conf: client: log file: /var/log/ceph/ceph-$name.$pid.log mon: {} install: ceph: flavor: default sha1: 8293d73f8690540e843a81caec373f9cc29cf705 selinux: allowlist: - scontext=system_u:system_r:logrotate_t:s0 workunit: branch: wip-guits-main-2024-09-13-1248 sha1: 8293d73f8690540e843a81caec373f9cc29cf705 owner: scheduled_gabrioux@teuthology priority: 70 repo: https://git.ceph.com/ceph-ci.git roles: - - host.a - osd.0 - osd.1 - osd.2 - osd.3 - client.0 - - host.b - osd.4 - osd.5 - osd.6 - osd.7 seed: 6714 sha1: 8293d73f8690540e843a81caec373f9cc29cf705 sleep_before_teardown: 0 subset: 1/8 suite: orch:cephadm suite_branch: wip-guits-main-2024-09-13-1248 suite_path: /home/teuthworker/src/git.ceph.com_ceph-c_8293d73f8690540e843a81caec373f9cc29cf705/qa suite_relpath: qa suite_repo: https://git.ceph.com/ceph-ci.git suite_sha1: 8293d73f8690540e843a81caec373f9cc29cf705 targets: smithi028.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBDbe09/m+wyDz7aQ5g27KdWS5PyxiztsRUCatu4pRbnlr+bdCCLE/wMYLcwLlUb+sCKVLGv1x7BWYxExdPs0Wq4= smithi144.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBJFTNKEa1EWzFm46IrSlwA/fJrJ+nGrBd5CQy5r+4trpxzXxJXH3FcnCh3oAcTQBoAtvK80S+w8YOkYuagl/uQs= tasks: - cephadm: cephadm_branch: v17.2.0 cephadm_git_url: https://github.com/ceph/ceph image: quay.io/ceph/ceph:v17.2.0 roleless: true - cephadm.shell: host.a: - ceph orch status - ceph orch ps - ceph orch ls - ceph orch host ls - ceph orch device ls - vip.exec: all-hosts: - systemctl stop nfs-server - cephadm.shell: host.a: - ceph fs volume create foofs - cephadm.wait_for_service: service: mds.foofs - cephadm.shell: host.a: - ceph nfs cluster create foo --placement=2 || ceph nfs cluster create cephfs foo --placement=2 - ceph nfs export create cephfs --fsname foofs --clusterid foo --binding /fake || ceph nfs export create cephfs --fsname foofs --cluster-id foo --pseudo-path /fake - while ! ceph orch ls | grep nfs | grep 2/2 ; do sleep 1 ; done - vip.exec: host.a: - mkdir /mnt/foo - while ! mount -t nfs $(hostname):/fake /mnt/foo -o sync ; do sleep 5 ; done - echo test > /mnt/foo/testfile - sync - parallel: - upgrade-tasks - workload-tasks - vip.exec: host.a: - umount /mnt/foo - cephadm.shell: host.a: - ceph nfs cluster ls | grep foo - ceph nfs export ls foo --detailed - rados -p .nfs --all ls - - cephadm.shell: host.a: - 'set -ex [[ `ceph config get mgr mgr/cephadm/migration_current` -gt 2 ]] ' teuthology: fragments_dropped: [] meta: {} postmerge: [] teuthology_branch: main teuthology_sha1: 3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a timestamp: 2024-09-16_07:44:22 tube: smithi upgrade-tasks: sequential: - cephadm.shell: env: - sha1 host.a: - ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force - ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force - ceph config set global log_to_journald false --force - ceph mgr module enable nfs --force - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 - cephadm.shell: env: - sha1 host.a: - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; ceph health detail ; sleep 30 ; done - ceph orch ps - ceph versions - echo "wait for servicemap items w/ changing names to refresh" - sleep 60 - ceph orch ps - ceph orch upgrade status - ceph health detail - ceph versions - ceph versions | jq -e '.overall | length == 1' - ceph versions | jq -e '.overall | keys' | grep $sha1 - cephadm.wait_for_service: service: nfs.foo user: gabrioux verbose: true worker_log: /home/teuthworker/archive/worker_logs/dispatcher.smithi.3401856 workload-tasks: sequential: - exec: host.a: - cd /mnt/foo && dbench 5 -t 600 || true - umount /mnt/foo - while ! mount -t nfs $(hostname):/fake /mnt/foo ; do sleep 5 ; done - cd /mnt/foo && dbench 5 -t 5 2024-09-16T07:57:45.439 INFO:teuthology.run:suite_path is set to /home/teuthworker/src/git.ceph.com_ceph-c_8293d73f8690540e843a81caec373f9cc29cf705/qa; will attempt to use it 2024-09-16T07:57:45.440 INFO:teuthology.run:Found tasks at /home/teuthworker/src/git.ceph.com_ceph-c_8293d73f8690540e843a81caec373f9cc29cf705/qa/tasks 2024-09-16T07:57:45.440 INFO:teuthology.run_tasks:Running task internal.check_packages... 2024-09-16T07:57:45.442 INFO:teuthology.task.internal:Checking packages... 2024-09-16T07:57:45.465 INFO:teuthology.task.internal:Checking packages for os_type 'centos', flavor 'default' and ceph hash '8293d73f8690540e843a81caec373f9cc29cf705' 2024-09-16T07:57:45.465 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-09-16T07:57:45.465 INFO:teuthology.packaging:ref: None 2024-09-16T07:57:45.466 INFO:teuthology.packaging:tag: None 2024-09-16T07:57:45.466 INFO:teuthology.packaging:branch: wip-guits-main-2024-09-13-1248 2024-09-16T07:57:45.466 INFO:teuthology.packaging:sha1: 8293d73f8690540e843a81caec373f9cc29cf705 2024-09-16T07:57:45.466 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&ref=wip-guits-main-2024-09-13-1248 2024-09-16T07:57:45.614 INFO:teuthology.task.internal:Found packages for ceph version 19.3.0-4917.g8293d73f 2024-09-16T07:57:45.616 INFO:teuthology.run_tasks:Running task internal.buildpackages_prep... 2024-09-16T07:57:45.621 INFO:teuthology.task.internal:no buildpackages task found 2024-09-16T07:57:45.621 INFO:teuthology.run_tasks:Running task internal.save_config... 2024-09-16T07:57:45.638 INFO:teuthology.task.internal:Saving configuration 2024-09-16T07:57:45.654 INFO:teuthology.run_tasks:Running task internal.check_lock... 2024-09-16T07:57:45.661 INFO:teuthology.task.internal.check_lock:Checking locks... 2024-09-16T07:57:45.684 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi028.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/gabrioux-2024-09-16_07:44:22-orch:cephadm-wip-guits-main-2024-09-13-1248-distro-default-smithi/7906916', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9.stream', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-09-16 07:52:45.545246', 'locked_by': 'scheduled_gabrioux@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBDbe09/m+wyDz7aQ5g27KdWS5PyxiztsRUCatu4pRbnlr+bdCCLE/wMYLcwLlUb+sCKVLGv1x7BWYxExdPs0Wq4='} 2024-09-16T07:57:45.710 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi144.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/gabrioux-2024-09-16_07:44:22-orch:cephadm-wip-guits-main-2024-09-13-1248-distro-default-smithi/7906916', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9.stream', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-09-16 07:52:45.543296', 'locked_by': 'scheduled_gabrioux@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBJFTNKEa1EWzFm46IrSlwA/fJrJ+nGrBd5CQy5r+4trpxzXxJXH3FcnCh3oAcTQBoAtvK80S+w8YOkYuagl/uQs='} 2024-09-16T07:57:45.710 INFO:teuthology.run_tasks:Running task internal.add_remotes... 2024-09-16T07:57:45.716 INFO:teuthology.task.internal:roles: ubuntu@smithi028.front.sepia.ceph.com - ['host.a', 'osd.0', 'osd.1', 'osd.2', 'osd.3', 'client.0'] 2024-09-16T07:57:45.716 INFO:teuthology.task.internal:roles: ubuntu@smithi144.front.sepia.ceph.com - ['host.b', 'osd.4', 'osd.5', 'osd.6', 'osd.7'] 2024-09-16T07:57:45.717 INFO:teuthology.run_tasks:Running task console_log... 2024-09-16T07:57:45.782 DEBUG:teuthology.exit:Installing handler: Handler(exiter=, func=.kill_console_loggers at 0x7fb1c19e3be0>, signals=[15]) 2024-09-16T07:57:45.782 INFO:teuthology.run_tasks:Running task internal.connect... 2024-09-16T07:57:45.788 INFO:teuthology.task.internal:Opening connections... 2024-09-16T07:57:45.788 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi028.front.sepia.ceph.com 2024-09-16T07:57:45.789 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi028.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-16T07:57:45.865 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi144.front.sepia.ceph.com 2024-09-16T07:57:45.866 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi144.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-16T07:57:45.941 INFO:teuthology.run_tasks:Running task internal.push_inventory... 2024-09-16T07:57:45.947 DEBUG:teuthology.orchestra.run.smithi028:> uname -m 2024-09-16T07:57:45.965 INFO:teuthology.orchestra.run.smithi028.stdout:x86_64 2024-09-16T07:57:45.966 DEBUG:teuthology.orchestra.run.smithi028:> cat /etc/os-release 2024-09-16T07:57:46.023 INFO:teuthology.orchestra.run.smithi028.stdout:NAME="CentOS Stream" 2024-09-16T07:57:46.023 INFO:teuthology.orchestra.run.smithi028.stdout:VERSION="9" 2024-09-16T07:57:46.023 INFO:teuthology.orchestra.run.smithi028.stdout:ID="centos" 2024-09-16T07:57:46.023 INFO:teuthology.orchestra.run.smithi028.stdout:ID_LIKE="rhel fedora" 2024-09-16T07:57:46.023 INFO:teuthology.orchestra.run.smithi028.stdout:VERSION_ID="9" 2024-09-16T07:57:46.023 INFO:teuthology.orchestra.run.smithi028.stdout:PLATFORM_ID="platform:el9" 2024-09-16T07:57:46.024 INFO:teuthology.orchestra.run.smithi028.stdout:PRETTY_NAME="CentOS Stream 9" 2024-09-16T07:57:46.024 INFO:teuthology.orchestra.run.smithi028.stdout:ANSI_COLOR="0;31" 2024-09-16T07:57:46.024 INFO:teuthology.orchestra.run.smithi028.stdout:LOGO="fedora-logo-icon" 2024-09-16T07:57:46.024 INFO:teuthology.orchestra.run.smithi028.stdout:CPE_NAME="cpe:/o:centos:centos:9" 2024-09-16T07:57:46.024 INFO:teuthology.orchestra.run.smithi028.stdout:HOME_URL="https://centos.org/" 2024-09-16T07:57:46.024 INFO:teuthology.orchestra.run.smithi028.stdout:BUG_REPORT_URL="https://issues.redhat.com/" 2024-09-16T07:57:46.024 INFO:teuthology.orchestra.run.smithi028.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 9" 2024-09-16T07:57:46.024 INFO:teuthology.orchestra.run.smithi028.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2024-09-16T07:57:46.025 INFO:teuthology.lock.ops:Updating smithi028.front.sepia.ceph.com on lock server 2024-09-16T07:57:46.046 DEBUG:teuthology.orchestra.run.smithi144:> uname -m 2024-09-16T07:57:46.065 INFO:teuthology.orchestra.run.smithi144.stdout:x86_64 2024-09-16T07:57:46.065 DEBUG:teuthology.orchestra.run.smithi144:> cat /etc/os-release 2024-09-16T07:57:46.123 INFO:teuthology.orchestra.run.smithi144.stdout:NAME="CentOS Stream" 2024-09-16T07:57:46.123 INFO:teuthology.orchestra.run.smithi144.stdout:VERSION="9" 2024-09-16T07:57:46.123 INFO:teuthology.orchestra.run.smithi144.stdout:ID="centos" 2024-09-16T07:57:46.123 INFO:teuthology.orchestra.run.smithi144.stdout:ID_LIKE="rhel fedora" 2024-09-16T07:57:46.123 INFO:teuthology.orchestra.run.smithi144.stdout:VERSION_ID="9" 2024-09-16T07:57:46.123 INFO:teuthology.orchestra.run.smithi144.stdout:PLATFORM_ID="platform:el9" 2024-09-16T07:57:46.123 INFO:teuthology.orchestra.run.smithi144.stdout:PRETTY_NAME="CentOS Stream 9" 2024-09-16T07:57:46.123 INFO:teuthology.orchestra.run.smithi144.stdout:ANSI_COLOR="0;31" 2024-09-16T07:57:46.123 INFO:teuthology.orchestra.run.smithi144.stdout:LOGO="fedora-logo-icon" 2024-09-16T07:57:46.124 INFO:teuthology.orchestra.run.smithi144.stdout:CPE_NAME="cpe:/o:centos:centos:9" 2024-09-16T07:57:46.124 INFO:teuthology.orchestra.run.smithi144.stdout:HOME_URL="https://centos.org/" 2024-09-16T07:57:46.124 INFO:teuthology.orchestra.run.smithi144.stdout:BUG_REPORT_URL="https://issues.redhat.com/" 2024-09-16T07:57:46.124 INFO:teuthology.orchestra.run.smithi144.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 9" 2024-09-16T07:57:46.124 INFO:teuthology.orchestra.run.smithi144.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2024-09-16T07:57:46.124 INFO:teuthology.lock.ops:Updating smithi144.front.sepia.ceph.com on lock server 2024-09-16T07:57:46.146 INFO:teuthology.run_tasks:Running task internal.serialize_remote_roles... 2024-09-16T07:57:46.155 INFO:teuthology.run_tasks:Running task internal.check_conflict... 2024-09-16T07:57:46.160 INFO:teuthology.task.internal:Checking for old test directory... 2024-09-16T07:57:46.160 DEBUG:teuthology.orchestra.run.smithi028:> test '!' -e /home/ubuntu/cephtest 2024-09-16T07:57:46.163 DEBUG:teuthology.orchestra.run.smithi144:> test '!' -e /home/ubuntu/cephtest 2024-09-16T07:57:46.179 INFO:teuthology.run_tasks:Running task internal.check_ceph_data... 2024-09-16T07:57:46.185 INFO:teuthology.task.internal:Checking for non-empty /var/lib/ceph... 2024-09-16T07:57:46.185 DEBUG:teuthology.orchestra.run.smithi028:> test -z $(ls -A /var/lib/ceph) 2024-09-16T07:57:46.220 DEBUG:teuthology.orchestra.run.smithi144:> test -z $(ls -A /var/lib/ceph) 2024-09-16T07:57:46.324 INFO:teuthology.run_tasks:Running task internal.vm_setup... 2024-09-16T07:57:46.373 INFO:teuthology.run_tasks:Running task kernel... 2024-09-16T07:57:46.387 INFO:teuthology.task.kernel:normalize config orig: {'kdb': 1, 'sha1': 'distro'} 2024-09-16T07:57:46.387 INFO:teuthology.task.kernel:config {'host.a': {'kdb': 1, 'sha1': 'distro'}, 'host.b': {'kdb': 1, 'sha1': 'distro'}}, timeout 300 2024-09-16T07:57:46.387 DEBUG:teuthology.orchestra.run.smithi028:> test -f /run/.containerenv -o -f /.dockerenv 2024-09-16T07:57:46.388 DEBUG:teuthology.orchestra.run.smithi144:> test -f /run/.containerenv -o -f /.dockerenv 2024-09-16T07:57:46.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-16T07:57:46.405 DEBUG:teuthology.orchestra.run.smithi028:> uname -r 2024-09-16T07:57:46.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-16T07:57:46.406 DEBUG:teuthology.orchestra.run.smithi144:> uname -r 2024-09-16T07:57:46.463 INFO:teuthology.orchestra.run.smithi028.stdout:5.14.0-503.el9.x86_64 2024-09-16T07:57:46.463 INFO:teuthology.task.kernel:Running kernel on smithi028: 5.14.0-503.el9.x86_64 2024-09-16T07:57:46.463 DEBUG:teuthology.orchestra.run.smithi028:> sudo yum install -y kernel 2024-09-16T07:57:46.464 INFO:teuthology.orchestra.run.smithi144.stdout:5.14.0-503.el9.x86_64 2024-09-16T07:57:46.465 INFO:teuthology.task.kernel:Running kernel on smithi144: 5.14.0-503.el9.x86_64 2024-09-16T07:57:46.465 DEBUG:teuthology.orchestra.run.smithi144:> sudo yum install -y kernel 2024-09-16T07:57:49.108 INFO:teuthology.orchestra.run.smithi144.stdout:CentOS Stream 9 - BaseOS 9.3 MB/s | 8.2 MB 00:00 2024-09-16T07:57:49.938 INFO:teuthology.orchestra.run.smithi028.stdout:CentOS Stream 9 - BaseOS 4.5 MB/s | 8.2 MB 00:01 2024-09-16T07:57:50.983 INFO:teuthology.orchestra.run.smithi144.stdout:CentOS Stream 9 - AppStream 27 MB/s | 20 MB 00:00 2024-09-16T07:57:56.317 INFO:teuthology.orchestra.run.smithi144.stdout:CentOS Stream 9 - CRB 4.4 MB/s | 6.5 MB 00:01 2024-09-16T07:57:57.684 INFO:teuthology.orchestra.run.smithi144.stdout:CentOS Stream 9 - Extras packages 125 kB/s | 19 kB 00:00 2024-09-16T07:57:58.672 INFO:teuthology.orchestra.run.smithi144.stdout:Extra Packages for Enterprise Linux 26 MB/s | 23 MB 00:00 2024-09-16T07:57:58.831 INFO:teuthology.orchestra.run.smithi028.stdout:CentOS Stream 9 - AppStream 2.6 MB/s | 20 MB 00:07 2024-09-16T07:58:03.440 INFO:teuthology.orchestra.run.smithi028.stdout:CentOS Stream 9 - CRB 9.9 MB/s | 6.5 MB 00:00 2024-09-16T07:58:04.095 INFO:teuthology.orchestra.run.smithi144.stdout:lab-extras 31 kB/s | 1.7 kB 00:00 2024-09-16T07:58:04.974 INFO:teuthology.orchestra.run.smithi028.stdout:CentOS Stream 9 - Extras packages 68 kB/s | 19 kB 00:00 2024-09-16T07:58:05.911 INFO:teuthology.orchestra.run.smithi028.stdout:Extra Packages for Enterprise Linux 28 MB/s | 23 MB 00:00 2024-09-16T07:58:06.134 INFO:teuthology.orchestra.run.smithi144.stdout:Package kernel-5.14.0-239.el9.x86_64 is already installed. 2024-09-16T07:58:06.135 INFO:teuthology.orchestra.run.smithi144.stdout:Package kernel-5.14.0-503.el9.x86_64 is already installed. 2024-09-16T07:58:06.233 INFO:teuthology.orchestra.run.smithi144.stdout:Dependencies resolved. 2024-09-16T07:58:06.237 INFO:teuthology.orchestra.run.smithi144.stdout:================================================================================ 2024-09-16T07:58:06.238 INFO:teuthology.orchestra.run.smithi144.stdout: Package Architecture Version Repository Size 2024-09-16T07:58:06.238 INFO:teuthology.orchestra.run.smithi144.stdout:================================================================================ 2024-09-16T07:58:06.238 INFO:teuthology.orchestra.run.smithi144.stdout:Installing: 2024-09-16T07:58:06.238 INFO:teuthology.orchestra.run.smithi144.stdout: kernel x86_64 5.14.0-505.el9 baseos 23 k 2024-09-16T07:58:06.238 INFO:teuthology.orchestra.run.smithi144.stdout:Installing dependencies: 2024-09-16T07:58:06.238 INFO:teuthology.orchestra.run.smithi144.stdout: kernel-core x86_64 5.14.0-505.el9 baseos 16 M 2024-09-16T07:58:06.238 INFO:teuthology.orchestra.run.smithi144.stdout: kernel-modules x86_64 5.14.0-505.el9 baseos 35 M 2024-09-16T07:58:06.239 INFO:teuthology.orchestra.run.smithi144.stdout: kernel-modules-core x86_64 5.14.0-505.el9 baseos 29 M 2024-09-16T07:58:06.239 INFO:teuthology.orchestra.run.smithi144.stdout: 2024-09-16T07:58:06.239 INFO:teuthology.orchestra.run.smithi144.stdout:Transaction Summary 2024-09-16T07:58:06.239 INFO:teuthology.orchestra.run.smithi144.stdout:================================================================================ 2024-09-16T07:58:06.239 INFO:teuthology.orchestra.run.smithi144.stdout:Install 4 Packages 2024-09-16T07:58:06.239 INFO:teuthology.orchestra.run.smithi144.stdout: 2024-09-16T07:58:06.240 INFO:teuthology.orchestra.run.smithi144.stdout:Total download size: 79 M 2024-09-16T07:58:06.240 INFO:teuthology.orchestra.run.smithi144.stdout:Installed size: 126 M 2024-09-16T07:58:06.240 INFO:teuthology.orchestra.run.smithi144.stdout:Downloading Packages: 2024-09-16T07:58:06.446 INFO:teuthology.orchestra.run.smithi144.stdout:(1/4): kernel-5.14.0-505.el9.x86_64.rpm 209 kB/s | 23 kB 00:00 2024-09-16T07:58:06.988 INFO:teuthology.orchestra.run.smithi144.stdout:(2/4): kernel-modules-5.14.0-505.el9.x86_64.rpm 53 MB/s | 35 MB 00:00 2024-09-16T07:58:07.305 INFO:teuthology.orchestra.run.smithi144.stdout:(3/4): kernel-modules-core-5.14.0-505.el9.x86_6 33 MB/s | 29 MB 00:00 2024-09-16T07:58:07.597 INFO:teuthology.orchestra.run.smithi144.stdout:(4/4): kernel-core-5.14.0-505.el9.x86_64.rpm 12 MB/s | 16 MB 00:01 2024-09-16T07:58:07.597 INFO:teuthology.orchestra.run.smithi144.stdout:-------------------------------------------------------------------------------- 2024-09-16T07:58:07.597 INFO:teuthology.orchestra.run.smithi144.stdout:Total 58 MB/s | 79 MB 00:01 2024-09-16T07:58:08.281 INFO:teuthology.orchestra.run.smithi144.stdout:Running transaction check 2024-09-16T07:58:08.651 INFO:teuthology.orchestra.run.smithi144.stdout:Transaction check succeeded. 2024-09-16T07:58:08.651 INFO:teuthology.orchestra.run.smithi144.stdout:Running transaction test 2024-09-16T07:58:09.590 INFO:teuthology.orchestra.run.smithi144.stdout:Transaction test succeeded. 2024-09-16T07:58:09.591 INFO:teuthology.orchestra.run.smithi144.stdout:Running transaction 2024-09-16T07:58:11.592 INFO:teuthology.orchestra.run.smithi028.stdout:lab-extras 31 kB/s | 1.7 kB 00:00 2024-09-16T07:58:12.058 INFO:teuthology.orchestra.run.smithi144.stdout: Preparing : 1/1 2024-09-16T07:58:12.767 INFO:teuthology.orchestra.run.smithi144.stdout: Installing : kernel-modules-core-5.14.0-505.el9.x86_64 1/4 2024-09-16T07:58:12.857 INFO:teuthology.orchestra.run.smithi144.stdout: Installing : kernel-core-5.14.0-505.el9.x86_64 2/4 2024-09-16T07:58:13.699 INFO:teuthology.orchestra.run.smithi028.stdout:Package kernel-5.14.0-239.el9.x86_64 is already installed. 2024-09-16T07:58:13.699 INFO:teuthology.orchestra.run.smithi028.stdout:Package kernel-5.14.0-503.el9.x86_64 is already installed. 2024-09-16T07:58:13.802 INFO:teuthology.orchestra.run.smithi028.stdout:Dependencies resolved. 2024-09-16T07:58:13.807 INFO:teuthology.orchestra.run.smithi028.stdout:================================================================================ 2024-09-16T07:58:13.807 INFO:teuthology.orchestra.run.smithi028.stdout: Package Architecture Version Repository Size 2024-09-16T07:58:13.807 INFO:teuthology.orchestra.run.smithi028.stdout:================================================================================ 2024-09-16T07:58:13.807 INFO:teuthology.orchestra.run.smithi028.stdout:Installing: 2024-09-16T07:58:13.807 INFO:teuthology.orchestra.run.smithi028.stdout: kernel x86_64 5.14.0-505.el9 baseos 23 k 2024-09-16T07:58:13.807 INFO:teuthology.orchestra.run.smithi028.stdout:Installing dependencies: 2024-09-16T07:58:13.807 INFO:teuthology.orchestra.run.smithi028.stdout: kernel-core x86_64 5.14.0-505.el9 baseos 16 M 2024-09-16T07:58:13.807 INFO:teuthology.orchestra.run.smithi028.stdout: kernel-modules x86_64 5.14.0-505.el9 baseos 35 M 2024-09-16T07:58:13.807 INFO:teuthology.orchestra.run.smithi028.stdout: kernel-modules-core x86_64 5.14.0-505.el9 baseos 29 M 2024-09-16T07:58:13.807 INFO:teuthology.orchestra.run.smithi028.stdout: 2024-09-16T07:58:13.807 INFO:teuthology.orchestra.run.smithi028.stdout:Transaction Summary 2024-09-16T07:58:13.807 INFO:teuthology.orchestra.run.smithi028.stdout:================================================================================ 2024-09-16T07:58:13.807 INFO:teuthology.orchestra.run.smithi028.stdout:Install 4 Packages 2024-09-16T07:58:13.808 INFO:teuthology.orchestra.run.smithi028.stdout: 2024-09-16T07:58:13.808 INFO:teuthology.orchestra.run.smithi028.stdout:Total download size: 79 M 2024-09-16T07:58:13.808 INFO:teuthology.orchestra.run.smithi028.stdout:Installed size: 126 M 2024-09-16T07:58:13.809 INFO:teuthology.orchestra.run.smithi028.stdout:Downloading Packages: 2024-09-16T07:58:13.935 INFO:teuthology.orchestra.run.smithi028.stdout:(1/4): kernel-5.14.0-505.el9.x86_64.rpm 215 kB/s | 23 kB 00:00 2024-09-16T07:58:14.885 INFO:teuthology.orchestra.run.smithi028.stdout:(2/4): kernel-core-5.14.0-505.el9.x86_64.rpm 15 MB/s | 16 MB 00:01 2024-09-16T07:58:15.369 INFO:teuthology.orchestra.run.smithi028.stdout:(3/4): kernel-modules-core-5.14.0-505.el9.x86_6 20 MB/s | 29 MB 00:01 2024-09-16T07:58:15.654 INFO:teuthology.orchestra.run.smithi144.stdout: Running scriptlet: kernel-core-5.14.0-505.el9.x86_64 2/4 2024-09-16T07:58:15.727 INFO:teuthology.orchestra.run.smithi028.stdout:(4/4): kernel-modules-5.14.0-505.el9.x86_64.rpm 18 MB/s | 35 MB 00:01 2024-09-16T07:58:15.728 INFO:teuthology.orchestra.run.smithi028.stdout:-------------------------------------------------------------------------------- 2024-09-16T07:58:15.728 INFO:teuthology.orchestra.run.smithi028.stdout:Total 41 MB/s | 79 MB 00:01 2024-09-16T07:58:15.784 INFO:teuthology.orchestra.run.smithi144.stdout: Installing : kernel-modules-5.14.0-505.el9.x86_64 3/4 2024-09-16T07:58:16.439 INFO:teuthology.orchestra.run.smithi028.stdout:Running transaction check 2024-09-16T07:58:16.808 INFO:teuthology.orchestra.run.smithi028.stdout:Transaction check succeeded. 2024-09-16T07:58:16.808 INFO:teuthology.orchestra.run.smithi028.stdout:Running transaction test 2024-09-16T07:58:17.802 INFO:teuthology.orchestra.run.smithi028.stdout:Transaction test succeeded. 2024-09-16T07:58:17.802 INFO:teuthology.orchestra.run.smithi028.stdout:Running transaction 2024-09-16T07:58:20.385 INFO:teuthology.orchestra.run.smithi028.stdout: Preparing : 1/1 2024-09-16T07:58:20.604 INFO:teuthology.orchestra.run.smithi144.stdout: Running scriptlet: kernel-modules-5.14.0-505.el9.x86_64 3/4 2024-09-16T07:58:20.726 INFO:teuthology.orchestra.run.smithi144.stdout: Installing : kernel-5.14.0-505.el9.x86_64 4/4 2024-09-16T07:58:21.055 INFO:teuthology.orchestra.run.smithi028.stdout: Installing : kernel-modules-core-5.14.0-505.el9.x86_64 1/4 2024-09-16T07:58:21.136 INFO:teuthology.orchestra.run.smithi028.stdout: Installing : kernel-core-5.14.0-505.el9.x86_64 2/4 2024-09-16T07:58:23.949 INFO:teuthology.orchestra.run.smithi028.stdout: Running scriptlet: kernel-core-5.14.0-505.el9.x86_64 2/4 2024-09-16T07:58:24.080 INFO:teuthology.orchestra.run.smithi028.stdout: Installing : kernel-modules-5.14.0-505.el9.x86_64 3/4 2024-09-16T07:58:25.585 INFO:teuthology.orchestra.run.smithi144.stdout: Running scriptlet: kernel-modules-core-5.14.0-505.el9.x86_64 4/4 2024-09-16T07:58:29.205 INFO:teuthology.orchestra.run.smithi028.stdout: Running scriptlet: kernel-modules-5.14.0-505.el9.x86_64 3/4 2024-09-16T07:58:29.248 INFO:teuthology.orchestra.run.smithi028.stdout: Installing : kernel-5.14.0-505.el9.x86_64 4/4 2024-09-16T07:58:34.388 INFO:teuthology.orchestra.run.smithi028.stdout: Running scriptlet: kernel-modules-core-5.14.0-505.el9.x86_64 4/4 2024-09-16T07:59:03.698 INFO:teuthology.orchestra.run.smithi144.stdout: Running scriptlet: kernel-core-5.14.0-505.el9.x86_64 4/4 2024-09-16T07:59:03.740 INFO:teuthology.orchestra.run.smithi144.stdout: Running scriptlet: kernel-modules-5.14.0-505.el9.x86_64 4/4 2024-09-16T07:59:04.148 INFO:teuthology.orchestra.run.smithi144.stdout: Running scriptlet: kernel-5.14.0-505.el9.x86_64 4/4 2024-09-16T07:59:04.148 INFO:teuthology.orchestra.run.smithi144.stdout: Verifying : kernel-5.14.0-505.el9.x86_64 1/4 2024-09-16T07:59:04.148 INFO:teuthology.orchestra.run.smithi144.stdout: Verifying : kernel-core-5.14.0-505.el9.x86_64 2/4 2024-09-16T07:59:04.149 INFO:teuthology.orchestra.run.smithi144.stdout: Verifying : kernel-modules-5.14.0-505.el9.x86_64 3/4 2024-09-16T07:59:05.042 INFO:teuthology.orchestra.run.smithi144.stdout: Verifying : kernel-modules-core-5.14.0-505.el9.x86_64 4/4 2024-09-16T07:59:05.050 INFO:teuthology.orchestra.run.smithi144.stdout: 2024-09-16T07:59:05.050 INFO:teuthology.orchestra.run.smithi144.stdout:Installed: 2024-09-16T07:59:05.050 INFO:teuthology.orchestra.run.smithi144.stdout: kernel-5.14.0-505.el9.x86_64 2024-09-16T07:59:05.050 INFO:teuthology.orchestra.run.smithi144.stdout: kernel-core-5.14.0-505.el9.x86_64 2024-09-16T07:59:05.050 INFO:teuthology.orchestra.run.smithi144.stdout: kernel-modules-5.14.0-505.el9.x86_64 2024-09-16T07:59:05.050 INFO:teuthology.orchestra.run.smithi144.stdout: kernel-modules-core-5.14.0-505.el9.x86_64 2024-09-16T07:59:05.050 INFO:teuthology.orchestra.run.smithi144.stdout: 2024-09-16T07:59:05.050 INFO:teuthology.orchestra.run.smithi144.stdout:Complete! 2024-09-16T07:59:05.391 DEBUG:teuthology.orchestra.run.smithi144:> rpm -q kernel | sort -rV | head -n 1 2024-09-16T07:59:05.471 INFO:teuthology.orchestra.run.smithi144.stdout:kernel-5.14.0-505.el9.x86_64 2024-09-16T07:59:05.472 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-505.el9.x86_64 2024-09-16T07:59:05.472 INFO:teuthology.task.kernel:Not newest distro kernel. Current: 5.14.0-503.el9.x86_64 Expected: 5.14.0-505.el9.x86_64 2024-09-16T07:59:05.472 INFO:teuthology.task.kernel:Skipping firmware on distro kernel 2024-09-16T07:59:05.472 DEBUG:teuthology.task.kernel:src is distro, skipping download 2024-09-16T07:59:05.472 INFO:teuthology.task.kernel:Installing distro kernel on host.b... 2024-09-16T07:59:05.473 DEBUG:teuthology.task.kernel:install_kernel(remote=ubuntu@smithi144.front.sepia.ceph.com, path=None, version=distro) 2024-09-16T07:59:05.473 DEBUG:teuthology.orchestra.run.smithi144:> rpm -q kernel | sort -rV | head -n 1 2024-09-16T07:59:05.532 INFO:teuthology.orchestra.run.smithi144.stdout:kernel-5.14.0-505.el9.x86_64 2024-09-16T07:59:05.532 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-505.el9.x86_64 2024-09-16T07:59:05.532 DEBUG:teuthology.orchestra.run.smithi144:> sudo rpm -qi grub2-tools 2024-09-16T07:59:05.564 INFO:teuthology.orchestra.run.smithi144.stdout:Name : grub2-tools 2024-09-16T07:59:05.564 INFO:teuthology.orchestra.run.smithi144.stdout:Epoch : 1 2024-09-16T07:59:05.564 INFO:teuthology.orchestra.run.smithi144.stdout:Version : 2.06 2024-09-16T07:59:05.564 INFO:teuthology.orchestra.run.smithi144.stdout:Release : 82.el9 2024-09-16T07:59:05.564 INFO:teuthology.orchestra.run.smithi144.stdout:Architecture: x86_64 2024-09-16T07:59:05.564 INFO:teuthology.orchestra.run.smithi144.stdout:Install Date: Wed 28 Aug 2024 09:06:27 PM UTC 2024-09-16T07:59:05.564 INFO:teuthology.orchestra.run.smithi144.stdout:Group : Unspecified 2024-09-16T07:59:05.564 INFO:teuthology.orchestra.run.smithi144.stdout:Size : 8274670 2024-09-16T07:59:05.564 INFO:teuthology.orchestra.run.smithi144.stdout:License : GPLv3+ 2024-09-16T07:59:05.565 INFO:teuthology.orchestra.run.smithi144.stdout:Signature : RSA/SHA256, Mon 01 Jul 2024 07:46:42 AM UTC, Key ID 05b555b38483c65d 2024-09-16T07:59:05.565 INFO:teuthology.orchestra.run.smithi144.stdout:Source RPM : grub2-2.06-82.el9.src.rpm 2024-09-16T07:59:05.565 INFO:teuthology.orchestra.run.smithi144.stdout:Build Date : Fri 28 Jun 2024 09:24:44 AM UTC 2024-09-16T07:59:05.565 INFO:teuthology.orchestra.run.smithi144.stdout:Build Host : x86-05.stream.rdu2.redhat.com 2024-09-16T07:59:05.565 INFO:teuthology.orchestra.run.smithi144.stdout:Packager : builder@centos.org 2024-09-16T07:59:05.565 INFO:teuthology.orchestra.run.smithi144.stdout:Vendor : CentOS 2024-09-16T07:59:05.565 INFO:teuthology.orchestra.run.smithi144.stdout:URL : http://www.gnu.org/software/grub/ 2024-09-16T07:59:05.565 INFO:teuthology.orchestra.run.smithi144.stdout:Summary : Support tools for GRUB. 2024-09-16T07:59:05.565 INFO:teuthology.orchestra.run.smithi144.stdout:Description : 2024-09-16T07:59:05.565 INFO:teuthology.orchestra.run.smithi144.stdout: 2024-09-16T07:59:05.565 INFO:teuthology.orchestra.run.smithi144.stdout:The GRand Unified Bootloader (GRUB) is a highly configurable and 2024-09-16T07:59:05.565 INFO:teuthology.orchestra.run.smithi144.stdout:customizable bootloader with modular architecture. It supports a rich 2024-09-16T07:59:05.565 INFO:teuthology.orchestra.run.smithi144.stdout:variety of kernel formats, file systems, computer architectures and 2024-09-16T07:59:05.566 INFO:teuthology.orchestra.run.smithi144.stdout:hardware devices. 2024-09-16T07:59:05.566 INFO:teuthology.orchestra.run.smithi144.stdout: 2024-09-16T07:59:05.566 INFO:teuthology.orchestra.run.smithi144.stdout:This subpackage provides tools for support of all platforms. 2024-09-16T07:59:05.567 INFO:teuthology.task.kernel:Updating Grub Version: grub2 2024-09-16T07:59:05.567 INFO:teuthology.task.kernel:Updating grub on smithi144 to boot 5.14.0-505.el9.x86_64 2024-09-16T07:59:05.567 DEBUG:teuthology.orchestra.run.smithi144:> sudo grub2-mkconfig -o /boot/grub2/grub.cfg 2024-09-16T07:59:06.246 INFO:teuthology.orchestra.run.smithi144.stderr:Generating grub configuration file ... 2024-09-16T07:59:07.570 INFO:teuthology.orchestra.run.smithi144.stderr:Adding boot menu entry for UEFI Firmware Settings ... 2024-09-16T07:59:07.599 INFO:teuthology.orchestra.run.smithi144.stderr:done 2024-09-16T07:59:07.601 DEBUG:teuthology.orchestra.run.smithi144:> mktemp 2024-09-16T07:59:07.616 INFO:teuthology.orchestra.run.smithi144.stdout:/tmp/tmp.LCAG1SjrFB 2024-09-16T07:59:07.617 DEBUG:teuthology.orchestra.run.smithi144:> sudo cp /boot/grub2/grub.cfg /tmp/tmp.LCAG1SjrFB 2024-09-16T07:59:07.684 DEBUG:teuthology.orchestra.run.smithi144:> sudo chmod 0666 /tmp/tmp.LCAG1SjrFB 2024-09-16T07:59:07.818 DEBUG:teuthology.orchestra.remote:smithi144:/tmp/tmp.LCAG1SjrFB is 6KB 2024-09-16T07:59:07.867 DEBUG:teuthology.orchestra.run.smithi144:> rm -fr /tmp/tmp.LCAG1SjrFB 2024-09-16T07:59:07.882 DEBUG:teuthology.orchestra.run.smithi144:> sudo /bin/ls /boot/loader/entries || true 2024-09-16T07:59:07.945 INFO:teuthology.orchestra.run.smithi144.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-0-rescue.conf 2024-09-16T07:59:07.945 INFO:teuthology.orchestra.run.smithi144.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-239.el9.x86_64.conf 2024-09-16T07:59:07.945 INFO:teuthology.orchestra.run.smithi144.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-503.el9.x86_64.conf 2024-09-16T07:59:07.946 INFO:teuthology.orchestra.run.smithi144.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-505.el9.x86_64.conf 2024-09-16T07:59:07.947 DEBUG:teuthology.orchestra.run.smithi144:> sudo grub2-set-default fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-505.el9.x86_64 2024-09-16T07:59:08.109 DEBUG:teuthology.orchestra.run.smithi144:> sudo shutdown -r now 2024-09-16T07:59:13.550 INFO:teuthology.orchestra.run.smithi028.stdout: Running scriptlet: kernel-core-5.14.0-505.el9.x86_64 4/4 2024-09-16T07:59:13.590 INFO:teuthology.orchestra.run.smithi028.stdout: Running scriptlet: kernel-modules-5.14.0-505.el9.x86_64 4/4 2024-09-16T07:59:13.912 INFO:teuthology.orchestra.run.smithi028.stdout: Running scriptlet: kernel-5.14.0-505.el9.x86_64 4/4 2024-09-16T07:59:13.912 INFO:teuthology.orchestra.run.smithi028.stdout: Verifying : kernel-5.14.0-505.el9.x86_64 1/4 2024-09-16T07:59:13.912 INFO:teuthology.orchestra.run.smithi028.stdout: Verifying : kernel-core-5.14.0-505.el9.x86_64 2/4 2024-09-16T07:59:13.912 INFO:teuthology.orchestra.run.smithi028.stdout: Verifying : kernel-modules-5.14.0-505.el9.x86_64 3/4 2024-09-16T07:59:14.804 INFO:teuthology.orchestra.run.smithi028.stdout: Verifying : kernel-modules-core-5.14.0-505.el9.x86_64 4/4 2024-09-16T07:59:14.804 INFO:teuthology.orchestra.run.smithi028.stdout: 2024-09-16T07:59:14.804 INFO:teuthology.orchestra.run.smithi028.stdout:Installed: 2024-09-16T07:59:14.804 INFO:teuthology.orchestra.run.smithi028.stdout: kernel-5.14.0-505.el9.x86_64 2024-09-16T07:59:14.804 INFO:teuthology.orchestra.run.smithi028.stdout: kernel-core-5.14.0-505.el9.x86_64 2024-09-16T07:59:14.804 INFO:teuthology.orchestra.run.smithi028.stdout: kernel-modules-5.14.0-505.el9.x86_64 2024-09-16T07:59:14.804 INFO:teuthology.orchestra.run.smithi028.stdout: kernel-modules-core-5.14.0-505.el9.x86_64 2024-09-16T07:59:14.804 INFO:teuthology.orchestra.run.smithi028.stdout: 2024-09-16T07:59:14.805 INFO:teuthology.orchestra.run.smithi028.stdout:Complete! 2024-09-16T07:59:15.145 DEBUG:teuthology.orchestra.run.smithi028:> rpm -q kernel | sort -rV | head -n 1 2024-09-16T07:59:15.231 INFO:teuthology.orchestra.run.smithi028.stdout:kernel-5.14.0-505.el9.x86_64 2024-09-16T07:59:15.231 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-505.el9.x86_64 2024-09-16T07:59:15.231 INFO:teuthology.task.kernel:Not newest distro kernel. Current: 5.14.0-503.el9.x86_64 Expected: 5.14.0-505.el9.x86_64 2024-09-16T07:59:15.232 INFO:teuthology.task.kernel:Skipping firmware on distro kernel 2024-09-16T07:59:15.232 DEBUG:teuthology.task.kernel:src is distro, skipping download 2024-09-16T07:59:15.232 INFO:teuthology.task.kernel:Installing distro kernel on host.a... 2024-09-16T07:59:15.232 DEBUG:teuthology.task.kernel:install_kernel(remote=ubuntu@smithi028.front.sepia.ceph.com, path=None, version=distro) 2024-09-16T07:59:15.232 DEBUG:teuthology.orchestra.run.smithi028:> rpm -q kernel | sort -rV | head -n 1 2024-09-16T07:59:15.291 INFO:teuthology.orchestra.run.smithi028.stdout:kernel-5.14.0-505.el9.x86_64 2024-09-16T07:59:15.291 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-505.el9.x86_64 2024-09-16T07:59:15.292 DEBUG:teuthology.orchestra.run.smithi028:> sudo rpm -qi grub2-tools 2024-09-16T07:59:15.332 INFO:teuthology.orchestra.run.smithi028.stdout:Name : grub2-tools 2024-09-16T07:59:15.332 INFO:teuthology.orchestra.run.smithi028.stdout:Epoch : 1 2024-09-16T07:59:15.332 INFO:teuthology.orchestra.run.smithi028.stdout:Version : 2.06 2024-09-16T07:59:15.332 INFO:teuthology.orchestra.run.smithi028.stdout:Release : 82.el9 2024-09-16T07:59:15.333 INFO:teuthology.orchestra.run.smithi028.stdout:Architecture: x86_64 2024-09-16T07:59:15.333 INFO:teuthology.orchestra.run.smithi028.stdout:Install Date: Wed 28 Aug 2024 09:06:27 PM UTC 2024-09-16T07:59:15.333 INFO:teuthology.orchestra.run.smithi028.stdout:Group : Unspecified 2024-09-16T07:59:15.333 INFO:teuthology.orchestra.run.smithi028.stdout:Size : 8274670 2024-09-16T07:59:15.333 INFO:teuthology.orchestra.run.smithi028.stdout:License : GPLv3+ 2024-09-16T07:59:15.333 INFO:teuthology.orchestra.run.smithi028.stdout:Signature : RSA/SHA256, Mon 01 Jul 2024 07:46:42 AM UTC, Key ID 05b555b38483c65d 2024-09-16T07:59:15.333 INFO:teuthology.orchestra.run.smithi028.stdout:Source RPM : grub2-2.06-82.el9.src.rpm 2024-09-16T07:59:15.333 INFO:teuthology.orchestra.run.smithi028.stdout:Build Date : Fri 28 Jun 2024 09:24:44 AM UTC 2024-09-16T07:59:15.333 INFO:teuthology.orchestra.run.smithi028.stdout:Build Host : x86-05.stream.rdu2.redhat.com 2024-09-16T07:59:15.333 INFO:teuthology.orchestra.run.smithi028.stdout:Packager : builder@centos.org 2024-09-16T07:59:15.334 INFO:teuthology.orchestra.run.smithi028.stdout:Vendor : CentOS 2024-09-16T07:59:15.334 INFO:teuthology.orchestra.run.smithi028.stdout:URL : http://www.gnu.org/software/grub/ 2024-09-16T07:59:15.334 INFO:teuthology.orchestra.run.smithi028.stdout:Summary : Support tools for GRUB. 2024-09-16T07:59:15.334 INFO:teuthology.orchestra.run.smithi028.stdout:Description : 2024-09-16T07:59:15.334 INFO:teuthology.orchestra.run.smithi028.stdout: 2024-09-16T07:59:15.334 INFO:teuthology.orchestra.run.smithi028.stdout:The GRand Unified Bootloader (GRUB) is a highly configurable and 2024-09-16T07:59:15.334 INFO:teuthology.orchestra.run.smithi028.stdout:customizable bootloader with modular architecture. It supports a rich 2024-09-16T07:59:15.335 INFO:teuthology.orchestra.run.smithi028.stdout:variety of kernel formats, file systems, computer architectures and 2024-09-16T07:59:15.335 INFO:teuthology.orchestra.run.smithi028.stdout:hardware devices. 2024-09-16T07:59:15.335 INFO:teuthology.orchestra.run.smithi028.stdout: 2024-09-16T07:59:15.335 INFO:teuthology.orchestra.run.smithi028.stdout:This subpackage provides tools for support of all platforms. 2024-09-16T07:59:15.336 INFO:teuthology.task.kernel:Updating Grub Version: grub2 2024-09-16T07:59:15.337 INFO:teuthology.task.kernel:Updating grub on smithi028 to boot 5.14.0-505.el9.x86_64 2024-09-16T07:59:15.337 DEBUG:teuthology.orchestra.run.smithi028:> sudo grub2-mkconfig -o /boot/grub2/grub.cfg 2024-09-16T07:59:16.010 INFO:teuthology.orchestra.run.smithi028.stderr:Generating grub configuration file ... 2024-09-16T07:59:17.348 INFO:teuthology.orchestra.run.smithi028.stderr:Adding boot menu entry for UEFI Firmware Settings ... 2024-09-16T07:59:17.391 INFO:teuthology.orchestra.run.smithi028.stderr:done 2024-09-16T07:59:17.393 DEBUG:teuthology.orchestra.run.smithi028:> mktemp 2024-09-16T07:59:17.408 INFO:teuthology.orchestra.run.smithi028.stdout:/tmp/tmp.v2I3E9AfnM 2024-09-16T07:59:17.408 DEBUG:teuthology.orchestra.run.smithi028:> sudo cp /boot/grub2/grub.cfg /tmp/tmp.v2I3E9AfnM 2024-09-16T07:59:17.479 DEBUG:teuthology.orchestra.run.smithi028:> sudo chmod 0666 /tmp/tmp.v2I3E9AfnM 2024-09-16T07:59:17.621 DEBUG:teuthology.orchestra.remote:smithi028:/tmp/tmp.v2I3E9AfnM is 6KB 2024-09-16T07:59:17.669 DEBUG:teuthology.orchestra.run.smithi028:> rm -fr /tmp/tmp.v2I3E9AfnM 2024-09-16T07:59:17.684 DEBUG:teuthology.orchestra.run.smithi028:> sudo /bin/ls /boot/loader/entries || true 2024-09-16T07:59:17.747 INFO:teuthology.orchestra.run.smithi028.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-0-rescue.conf 2024-09-16T07:59:17.747 INFO:teuthology.orchestra.run.smithi028.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-239.el9.x86_64.conf 2024-09-16T07:59:17.747 INFO:teuthology.orchestra.run.smithi028.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-503.el9.x86_64.conf 2024-09-16T07:59:17.747 INFO:teuthology.orchestra.run.smithi028.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-505.el9.x86_64.conf 2024-09-16T07:59:17.748 DEBUG:teuthology.orchestra.run.smithi028:> sudo grub2-set-default fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-505.el9.x86_64 2024-09-16T07:59:17.875 DEBUG:teuthology.orchestra.run.smithi028:> sudo shutdown -r now 2024-09-16T07:59:38.129 INFO:teuthology.task.kernel:Checking client host.b for new kernel version... 2024-09-16T07:59:38.130 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi144.front.sepia.ceph.com' 2024-09-16T07:59:38.132 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi144.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-16T07:59:47.882 INFO:teuthology.task.kernel:Checking client host.a for new kernel version... 2024-09-16T07:59:47.882 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi028.front.sepia.ceph.com' 2024-09-16T07:59:47.883 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi028.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-16T08:00:12.605 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.144 2024-09-16T08:00:21.613 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi144.front.sepia.ceph.com' 2024-09-16T08:00:21.614 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi144.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-16T08:00:39.997 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.144 2024-09-16T08:00:47.887 DEBUG:teuthology.orchestra.remote:timed out 2024-09-16T08:00:52.001 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi144.front.sepia.ceph.com' 2024-09-16T08:00:52.002 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi144.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-16T08:00:52.325 DEBUG:teuthology.orchestra.run.smithi144:> true 2024-09-16T08:00:52.652 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi144.front.sepia.ceph.com' 2024-09-16T08:00:52.652 INFO:teuthology.task.kernel:Checking kernel version of host.b, want "5.14.0-505.el9.x86_64"... 2024-09-16T08:00:52.653 DEBUG:teuthology.orchestra.run.smithi144:> uname -r 2024-09-16T08:00:52.667 INFO:teuthology.orchestra.run.smithi144.stdout:5.14.0-505.el9.x86_64 2024-09-16T08:00:52.667 DEBUG:teuthology.task.kernel:current kernel version is 5.14.0-505.el9.x86_64 vs 5.14.0-505.el9.x86_64 2024-09-16T08:00:52.667 DEBUG:teuthology.task.kernel:utsrelease strings match, do not need to install 2024-09-16T08:00:52.668 DEBUG:teuthology.task.kernel:Distro of this test job: centos 2024-09-16T08:00:53.668 INFO:teuthology.task.kernel:Enabling kdb on host.b... 2024-09-16T08:00:53.668 DEBUG:teuthology.orchestra.run.smithi144:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2024-09-16T08:00:53.776 INFO:teuthology.orchestra.run.smithi144.stdout:ttyS1 2024-09-16T08:00:53.803 DEBUG:teuthology.parallel:result is None 2024-09-16T08:00:56.889 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi028.front.sepia.ceph.com' 2024-09-16T08:00:56.889 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi028.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-16T08:00:57.217 DEBUG:teuthology.orchestra.run.smithi028:> true 2024-09-16T08:00:57.574 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi028.front.sepia.ceph.com' 2024-09-16T08:00:57.574 INFO:teuthology.task.kernel:Checking kernel version of host.a, want "5.14.0-505.el9.x86_64"... 2024-09-16T08:00:57.575 DEBUG:teuthology.orchestra.run.smithi028:> uname -r 2024-09-16T08:00:57.591 INFO:teuthology.orchestra.run.smithi028.stdout:5.14.0-505.el9.x86_64 2024-09-16T08:00:57.591 DEBUG:teuthology.task.kernel:current kernel version is 5.14.0-505.el9.x86_64 vs 5.14.0-505.el9.x86_64 2024-09-16T08:00:57.592 DEBUG:teuthology.task.kernel:utsrelease strings match, do not need to install 2024-09-16T08:00:57.592 DEBUG:teuthology.task.kernel:Distro of this test job: centos 2024-09-16T08:00:58.592 INFO:teuthology.task.kernel:Enabling kdb on host.a... 2024-09-16T08:00:58.592 DEBUG:teuthology.orchestra.run.smithi028:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2024-09-16T08:00:58.661 INFO:teuthology.orchestra.run.smithi028.stdout:ttyS1 2024-09-16T08:00:58.686 DEBUG:teuthology.parallel:result is None 2024-09-16T08:00:58.686 INFO:teuthology.run_tasks:Running task internal.base... 2024-09-16T08:00:58.694 INFO:teuthology.task.internal:Creating test directory... 2024-09-16T08:00:58.694 DEBUG:teuthology.orchestra.run.smithi028:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2024-09-16T08:00:58.696 DEBUG:teuthology.orchestra.run.smithi144:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2024-09-16T08:00:58.716 INFO:teuthology.run_tasks:Running task internal.archive_upload... 2024-09-16T08:00:58.724 INFO:teuthology.run_tasks:Running task internal.archive... 2024-09-16T08:00:58.729 INFO:teuthology.task.internal:Creating archive directory... 2024-09-16T08:00:58.730 DEBUG:teuthology.orchestra.run.smithi028:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2024-09-16T08:00:58.755 DEBUG:teuthology.orchestra.run.smithi144:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2024-09-16T08:00:58.791 INFO:teuthology.run_tasks:Running task internal.coredump... 2024-09-16T08:00:58.797 INFO:teuthology.task.internal:Enabling coredump saving... 2024-09-16T08:00:58.797 DEBUG:teuthology.orchestra.run.smithi028:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2024-09-16T08:00:58.827 DEBUG:teuthology.orchestra.run.smithi144:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2024-09-16T08:00:58.858 INFO:teuthology.orchestra.run.smithi144.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-09-16T08:00:58.858 INFO:teuthology.orchestra.run.smithi028.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-09-16T08:00:58.868 INFO:teuthology.orchestra.run.smithi144.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-09-16T08:00:58.870 INFO:teuthology.orchestra.run.smithi028.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-09-16T08:00:58.872 INFO:teuthology.run_tasks:Running task internal.sudo... 2024-09-16T08:00:58.879 INFO:teuthology.task.internal:Configuring sudo... 2024-09-16T08:00:58.879 DEBUG:teuthology.orchestra.run.smithi028:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2024-09-16T08:00:58.915 DEBUG:teuthology.orchestra.run.smithi144:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2024-09-16T08:00:58.943 INFO:teuthology.run_tasks:Running task internal.syslog... 2024-09-16T08:00:58.951 INFO:teuthology.task.internal.syslog:Starting syslog monitoring... 2024-09-16T08:00:58.952 DEBUG:teuthology.orchestra.run.smithi028:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2024-09-16T08:00:58.984 DEBUG:teuthology.orchestra.run.smithi144:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2024-09-16T08:00:59.000 DEBUG:teuthology.orchestra.run.smithi028:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2024-09-16T08:00:59.080 DEBUG:teuthology.orchestra.run.smithi028:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2024-09-16T08:00:59.140 DEBUG:teuthology.orchestra.run.smithi028:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2024-09-16T08:00:59.219 DEBUG:teuthology.orchestra.run.smithi028:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2024-09-16T08:00:59.268 DEBUG:teuthology.orchestra.run.smithi028:> set -ex 2024-09-16T08:00:59.268 DEBUG:teuthology.orchestra.run.smithi028:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2024-09-16T08:00:59.335 DEBUG:teuthology.orchestra.run.smithi144:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2024-09-16T08:00:59.375 DEBUG:teuthology.orchestra.run.smithi144:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2024-09-16T08:00:59.428 DEBUG:teuthology.orchestra.run.smithi144:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2024-09-16T08:00:59.508 DEBUG:teuthology.orchestra.run.smithi144:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2024-09-16T08:00:59.554 DEBUG:teuthology.orchestra.run.smithi144:> set -ex 2024-09-16T08:00:59.554 DEBUG:teuthology.orchestra.run.smithi144:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2024-09-16T08:00:59.620 DEBUG:teuthology.orchestra.run.smithi028:> sudo service rsyslog restart 2024-09-16T08:00:59.623 DEBUG:teuthology.orchestra.run.smithi144:> sudo service rsyslog restart 2024-09-16T08:00:59.670 INFO:teuthology.orchestra.run.smithi028.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-09-16T08:00:59.706 INFO:teuthology.orchestra.run.smithi144.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-09-16T08:01:00.217 INFO:teuthology.run_tasks:Running task internal.timer... 2024-09-16T08:01:00.225 INFO:teuthology.task.internal:Starting timer... 2024-09-16T08:01:00.225 INFO:teuthology.run_tasks:Running task pcp... 2024-09-16T08:01:00.236 INFO:teuthology.run_tasks:Running task selinux... 2024-09-16T08:01:00.245 DEBUG:teuthology.task:Applying overrides for task selinux: {'allowlist': ['scontext=system_u:system_r:logrotate_t:s0']} 2024-09-16T08:01:00.245 DEBUG:teuthology.orchestra.run.smithi028:> sudo service auditd rotate 2024-09-16T08:01:00.284 INFO:teuthology.orchestra.run.smithi028.stdout:Rotating logs: 2024-09-16T08:01:00.285 DEBUG:teuthology.orchestra.run.smithi144:> sudo service auditd rotate 2024-09-16T08:01:00.349 INFO:teuthology.orchestra.run.smithi144.stdout:Rotating logs: 2024-09-16T08:01:00.351 DEBUG:teuthology.task.selinux:Getting current SELinux state 2024-09-16T08:01:00.351 DEBUG:teuthology.orchestra.run.smithi028:> /usr/sbin/getenforce 2024-09-16T08:01:00.374 INFO:teuthology.orchestra.run.smithi028.stdout:Permissive 2024-09-16T08:01:00.374 DEBUG:teuthology.orchestra.run.smithi144:> /usr/sbin/getenforce 2024-09-16T08:01:00.397 INFO:teuthology.orchestra.run.smithi144.stdout:Permissive 2024-09-16T08:01:00.397 DEBUG:teuthology.task.selinux:Existing SELinux modes: {'ubuntu@smithi028.front.sepia.ceph.com': 'permissive', 'ubuntu@smithi144.front.sepia.ceph.com': 'permissive'} 2024-09-16T08:01:00.397 DEBUG:teuthology.orchestra.run.smithi028:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e context=system_u:system_r:getty_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 2024-09-16T08:01:00.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-16T08:01:00.443 DEBUG:teuthology.orchestra.run.smithi144:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e context=system_u:system_r:getty_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 2024-09-16T08:01:00.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-16T08:01:00.469 INFO:teuthology.task.selinux:Putting SELinux into permissive mode 2024-09-16T08:01:00.469 DEBUG:teuthology.orchestra.run.smithi028:> sudo /usr/sbin/setenforce permissive 2024-09-16T08:01:00.517 DEBUG:teuthology.orchestra.run.smithi144:> sudo /usr/sbin/setenforce permissive 2024-09-16T08:01:00.554 INFO:teuthology.run_tasks:Running task ansible.cephlab... 2024-09-16T08:01:00.564 DEBUG:teuthology.repo_utils:Resetting repo at /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main to origin/main 2024-09-16T08:01:00.575 INFO:teuthology.task.ansible:Playbook: [{'import_playbook': 'ansible_managed.yml'}, {'import_playbook': 'teuthology.yml'}, {'hosts': 'testnodes', 'tasks': [{'set_fact': {'ran_from_cephlab_playbook': True}}]}, {'import_playbook': 'testnodes.yml'}, {'import_playbook': 'container-host.yml'}, {'import_playbook': 'cobbler.yml'}, {'import_playbook': 'paddles.yml'}, {'import_playbook': 'pulpito.yml'}, {'hosts': 'testnodes', 'become': True, 'tasks': [{'name': 'Touch /ceph-qa-ready', 'file': {'path': '/ceph-qa-ready', 'state': 'touch'}, 'when': 'ran_from_cephlab_playbook|bool'}]}] 2024-09-16T08:01:00.576 DEBUG:teuthology.task.ansible:Running ansible-playbook -v --extra-vars '{"ansible_ssh_user": "ubuntu"}' -i /etc/ansible/hosts --limit smithi028.front.sepia.ceph.com,smithi144.front.sepia.ceph.com /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main/cephlab.yml 2024-09-16T08:04:44.694 DEBUG:teuthology.task.ansible:Reconnecting to [Remote(name='ubuntu@smithi028.front.sepia.ceph.com'), Remote(name='ubuntu@smithi144.front.sepia.ceph.com')] 2024-09-16T08:04:44.696 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi028.front.sepia.ceph.com' 2024-09-16T08:04:44.697 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi028.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-16T08:04:44.773 DEBUG:teuthology.orchestra.run.smithi028:> true 2024-09-16T08:04:44.852 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi028.front.sepia.ceph.com' 2024-09-16T08:04:44.853 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi144.front.sepia.ceph.com' 2024-09-16T08:04:44.853 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi144.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-16T08:04:44.928 DEBUG:teuthology.orchestra.run.smithi144:> true 2024-09-16T08:04:45.011 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi144.front.sepia.ceph.com' 2024-09-16T08:04:45.011 INFO:teuthology.run_tasks:Running task clock... 2024-09-16T08:04:45.021 INFO:teuthology.task.clock:Syncing clocks and checking initial clock skew... 2024-09-16T08:04:45.022 INFO:teuthology.orchestra.run:Running command with timeout 360 2024-09-16T08:04:45.022 DEBUG:teuthology.orchestra.run.smithi028:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-09-16T08:04:45.025 INFO:teuthology.orchestra.run:Running command with timeout 360 2024-09-16T08:04:45.025 DEBUG:teuthology.orchestra.run.smithi144:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-09-16T08:04:45.053 INFO:teuthology.orchestra.run.smithi028.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2024-09-16T08:04:45.068 INFO:teuthology.orchestra.run.smithi028.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2024-09-16T08:04:45.084 INFO:teuthology.orchestra.run.smithi144.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2024-09-16T08:04:45.094 INFO:teuthology.orchestra.run.smithi028.stderr:sudo: ntpd: command not found 2024-09-16T08:04:45.100 INFO:teuthology.orchestra.run.smithi144.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2024-09-16T08:04:45.105 INFO:teuthology.orchestra.run.smithi028.stdout:506 Cannot talk to daemon 2024-09-16T08:04:45.121 INFO:teuthology.orchestra.run.smithi028.stderr:Failed to start ntp.service: Unit ntp.service not found. 2024-09-16T08:04:45.126 INFO:teuthology.orchestra.run.smithi144.stderr:sudo: ntpd: command not found 2024-09-16T08:04:45.135 INFO:teuthology.orchestra.run.smithi028.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2024-09-16T08:04:45.139 INFO:teuthology.orchestra.run.smithi144.stdout:506 Cannot talk to daemon 2024-09-16T08:04:45.152 INFO:teuthology.orchestra.run.smithi144.stderr:Failed to start ntp.service: Unit ntp.service not found. 2024-09-16T08:04:45.167 INFO:teuthology.orchestra.run.smithi144.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2024-09-16T08:04:45.193 INFO:teuthology.orchestra.run.smithi028.stderr:bash: line 1: ntpq: command not found 2024-09-16T08:04:45.197 INFO:teuthology.orchestra.run.smithi028.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-09-16T08:04:45.197 INFO:teuthology.orchestra.run.smithi028.stdout:=============================================================================== 2024-09-16T08:04:45.197 INFO:teuthology.orchestra.run.smithi028.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-16T08:04:45.197 INFO:teuthology.orchestra.run.smithi028.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-16T08:04:45.197 INFO:teuthology.orchestra.run.smithi028.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-16T08:04:45.197 INFO:teuthology.orchestra.run.smithi028.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-16T08:04:45.222 INFO:teuthology.orchestra.run.smithi144.stderr:bash: line 1: ntpq: command not found 2024-09-16T08:04:45.225 INFO:teuthology.orchestra.run.smithi144.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-09-16T08:04:45.225 INFO:teuthology.orchestra.run.smithi144.stdout:=============================================================================== 2024-09-16T08:04:45.225 INFO:teuthology.orchestra.run.smithi144.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-16T08:04:45.226 INFO:teuthology.orchestra.run.smithi144.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-16T08:04:45.226 INFO:teuthology.orchestra.run.smithi144.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-16T08:04:45.226 INFO:teuthology.orchestra.run.smithi144.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-16T08:04:45.227 INFO:teuthology.run_tasks:Running task cephadm... 2024-09-16T08:04:45.563 INFO:tasks.cephadm:Config: {'cephadm_branch': 'v17.2.0', 'cephadm_git_url': 'https://github.com/ceph/ceph', 'image': 'quay.io/ceph/ceph:v17.2.0', 'roleless': True, 'conf': {'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'debug ms': 1, 'debug osd': 20, 'osd shutdown pgref assert': True}}, 'flavor': 'default', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)', 'CEPHADM_REFRESH_FAILED'], 'log-only-match': ['CEPHADM_'], 'sha1': '8293d73f8690540e843a81caec373f9cc29cf705'} 2024-09-16T08:04:45.563 INFO:tasks.cephadm:Cluster image is quay.io/ceph/ceph:v17.2.0 2024-09-16T08:04:45.564 INFO:tasks.cephadm:Cluster fsid is 5622b2fe-7402-11ef-bceb-c7b262605968 2024-09-16T08:04:45.564 INFO:tasks.cephadm:Choosing monitor IPs and ports... 2024-09-16T08:04:45.565 INFO:tasks.cephadm:No mon roles; fabricating mons 2024-09-16T08:04:45.565 INFO:tasks.cephadm:Monitor IPs: {'mon.smithi028': '172.21.15.28', 'mon.smithi144': '172.21.15.144'} 2024-09-16T08:04:45.565 INFO:tasks.cephadm:Normalizing hostnames... 2024-09-16T08:04:45.565 DEBUG:teuthology.orchestra.run.smithi028:> sudo hostname $(hostname -s) 2024-09-16T08:04:45.593 DEBUG:teuthology.orchestra.run.smithi144:> sudo hostname $(hostname -s) 2024-09-16T08:04:45.620 INFO:tasks.cephadm:Downloading cephadm (repo https://github.com/ceph/ceph ref v17.2.0)... 2024-09-16T08:04:45.620 DEBUG:teuthology.orchestra.run.smithi028:> curl --silent https://raw.githubusercontent.com/ceph/ceph/v17.2.0/src/cephadm/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2024-09-16T08:04:45.898 INFO:teuthology.orchestra.run.smithi028.stdout:-rw-r--r--. 1 ubuntu ubuntu 320521 Sep 16 08:04 /home/ubuntu/cephtest/cephadm 2024-09-16T08:04:45.898 DEBUG:teuthology.orchestra.run.smithi144:> curl --silent https://raw.githubusercontent.com/ceph/ceph/v17.2.0/src/cephadm/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2024-09-16T08:04:46.020 INFO:teuthology.orchestra.run.smithi144.stdout:-rw-r--r--. 1 ubuntu ubuntu 320521 Sep 16 08:04 /home/ubuntu/cephtest/cephadm 2024-09-16T08:04:46.020 DEBUG:teuthology.orchestra.run.smithi028:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2024-09-16T08:04:46.038 DEBUG:teuthology.orchestra.run.smithi144:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2024-09-16T08:04:46.063 INFO:tasks.cephadm:Pulling image quay.io/ceph/ceph:v17.2.0 on all hosts... 2024-09-16T08:04:46.063 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 pull 2024-09-16T08:04:46.081 DEBUG:teuthology.orchestra.run.smithi144:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 pull 2024-09-16T08:04:47.540 INFO:teuthology.orchestra.run.smithi028.stderr:Pulling container image quay.io/ceph/ceph:v17.2.0... 2024-09-16T08:04:47.704 INFO:teuthology.orchestra.run.smithi144.stderr:Pulling container image quay.io/ceph/ceph:v17.2.0... 2024-09-16T08:05:00.268 INFO:teuthology.orchestra.run.smithi144.stdout:{ 2024-09-16T08:05:00.268 INFO:teuthology.orchestra.run.smithi144.stdout: "ceph_version": "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)", 2024-09-16T08:05:00.269 INFO:teuthology.orchestra.run.smithi144.stdout: "image_id": "e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9", 2024-09-16T08:05:00.269 INFO:teuthology.orchestra.run.smithi144.stdout: "repo_digests": [ 2024-09-16T08:05:00.269 INFO:teuthology.orchestra.run.smithi144.stdout: "quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a", 2024-09-16T08:05:00.269 INFO:teuthology.orchestra.run.smithi144.stdout: "quay.io/ceph/ceph@sha256:cb4d698cb769b6aba05bf6ef04f41a7fe694160140347576e13bd9348514b667" 2024-09-16T08:05:00.269 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2024-09-16T08:05:00.269 INFO:teuthology.orchestra.run.smithi144.stdout:} 2024-09-16T08:05:01.238 INFO:teuthology.orchestra.run.smithi028.stdout:{ 2024-09-16T08:05:01.239 INFO:teuthology.orchestra.run.smithi028.stdout: "ceph_version": "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)", 2024-09-16T08:05:01.239 INFO:teuthology.orchestra.run.smithi028.stdout: "image_id": "e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9", 2024-09-16T08:05:01.239 INFO:teuthology.orchestra.run.smithi028.stdout: "repo_digests": [ 2024-09-16T08:05:01.239 INFO:teuthology.orchestra.run.smithi028.stdout: "quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a", 2024-09-16T08:05:01.239 INFO:teuthology.orchestra.run.smithi028.stdout: "quay.io/ceph/ceph@sha256:cb4d698cb769b6aba05bf6ef04f41a7fe694160140347576e13bd9348514b667" 2024-09-16T08:05:01.239 INFO:teuthology.orchestra.run.smithi028.stdout: ] 2024-09-16T08:05:01.239 INFO:teuthology.orchestra.run.smithi028.stdout:} 2024-09-16T08:05:01.257 DEBUG:teuthology.orchestra.run.smithi028:> sudo mkdir -p /etc/ceph 2024-09-16T08:05:01.296 DEBUG:teuthology.orchestra.run.smithi144:> sudo mkdir -p /etc/ceph 2024-09-16T08:05:01.329 DEBUG:teuthology.orchestra.run.smithi028:> sudo chmod 777 /etc/ceph 2024-09-16T08:05:01.361 DEBUG:teuthology.orchestra.run.smithi144:> sudo chmod 777 /etc/ceph 2024-09-16T08:05:01.400 INFO:tasks.cephadm:Writing seed config... 2024-09-16T08:05:01.403 INFO:tasks.cephadm: override: [mgr] debug mgr = 20 2024-09-16T08:05:01.403 INFO:tasks.cephadm: override: [mgr] debug ms = 1 2024-09-16T08:05:01.403 INFO:tasks.cephadm: override: [mon] debug mon = 20 2024-09-16T08:05:01.403 INFO:tasks.cephadm: override: [mon] debug ms = 1 2024-09-16T08:05:01.403 INFO:tasks.cephadm: override: [mon] debug paxos = 20 2024-09-16T08:05:01.404 INFO:tasks.cephadm: override: [osd] debug ms = 1 2024-09-16T08:05:01.404 INFO:tasks.cephadm: override: [osd] debug osd = 20 2024-09-16T08:05:01.404 INFO:tasks.cephadm: override: [osd] osd shutdown pgref assert = True 2024-09-16T08:05:01.404 DEBUG:teuthology.orchestra.run.smithi028:> set -ex 2024-09-16T08:05:01.404 DEBUG:teuthology.orchestra.run.smithi028:> dd of=/home/ubuntu/cephtest/seed.ceph.conf 2024-09-16T08:05:01.421 DEBUG:tasks.cephadm:Final config: [global] # make logging friendly to teuthology log_to_file = true log_to_stderr = false log to journald = false mon cluster log to file = true mon cluster log file level = debug mon clock drift allowed = 1.000 # replicate across OSDs, not hosts osd crush chooseleaf type = 0 #osd pool default size = 2 osd pool default erasure code profile = plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd # enable some debugging auth debug = true ms die on old message = true ms die on bug = true debug asserts on shutdown = true # adjust warnings mon max pg per osd = 10000# >= luminous mon pg warn max object skew = 0 mon osd allow primary affinity = true mon osd allow pg remap = true mon warn on legacy crush tunables = false mon warn on crush straw calc version zero = false mon warn on no sortbitwise = false mon warn on osd down out interval zero = false mon warn on too few osds = false mon_warn_on_pool_pg_num_not_power_of_two = false # disable pg_autoscaler by default for new pools osd_pool_default_pg_autoscale_mode = off # tests delete pools mon allow pool delete = true fsid = 5622b2fe-7402-11ef-bceb-c7b262605968 [osd] osd scrub load threshold = 5.0 osd scrub max interval = 600 osd mclock profile = high_recovery_ops osd recover clone overlap = true osd recovery max chunk = 1048576 osd deep scrub update digest min age = 30 osd map max advance = 10 osd memory target autotune = true # debugging osd debug shutdown = true osd debug op order = true osd debug verify stray on activate = true osd debug pg log writeout = true osd debug verify cached snaps = true osd debug verify missing on start = true osd debug misdirected ops = true osd op queue = debug_random osd op queue cut off = debug_random osd shutdown pgref assert = True bdev debug aio = true osd sloppy crc = true debug ms = 1 debug osd = 20 [mgr] mon reweight min pgs per osd = 4 mon reweight min bytes per osd = 10 mgr/telemetry/nag = false debug mgr = 20 debug ms = 1 [mon] mon data avail warn = 5 mon mgr mkfs grace = 240 mon reweight min pgs per osd = 4 mon osd reporter subtree level = osd mon osd prime pg temp = true mon reweight min bytes per osd = 10 # rotate auth tickets quickly to exercise renewal paths auth mon ticket ttl = 660# 11m auth service ticket ttl = 240# 4m # don't complain about global id reclaim mon_warn_on_insecure_global_id_reclaim = false mon_warn_on_insecure_global_id_reclaim_allowed = false debug mon = 20 debug ms = 1 debug paxos = 20 [client.rgw] rgw cache enabled = true rgw enable ops log = true rgw enable usage log = true 2024-09-16T08:05:01.421 DEBUG:teuthology.orchestra.run.smithi028:mon.smithi028> sudo journalctl -f -n 0 -u ceph-5622b2fe-7402-11ef-bceb-c7b262605968@mon.smithi028.service 2024-09-16T08:05:01.463 INFO:tasks.cephadm:Bootstrapping... 2024-09-16T08:05:01.463 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 -v bootstrap --fsid 5622b2fe-7402-11ef-bceb-c7b262605968 --config /home/ubuntu/cephtest/seed.ceph.conf --output-config /etc/ceph/ceph.conf --output-keyring /etc/ceph/ceph.client.admin.keyring --output-pub-ssh-key /home/ubuntu/cephtest/ceph.pub --mon-ip 172.21.15.28 --skip-admin-label && sudo chmod +r /etc/ceph/ceph.client.admin.keyring 2024-09-16T08:05:01.664 INFO:teuthology.orchestra.run.smithi028.stderr:-------------------------------------------------------------------------------- 2024-09-16T08:05:01.664 INFO:teuthology.orchestra.run.smithi028.stderr:cephadm ['--image', 'quay.io/ceph/ceph:v17.2.0', '-v', 'bootstrap', '--fsid', '5622b2fe-7402-11ef-bceb-c7b262605968', '--config', '/home/ubuntu/cephtest/seed.ceph.conf', '--output-config', '/etc/ceph/ceph.conf', '--output-keyring', '/etc/ceph/ceph.client.admin.keyring', '--output-pub-ssh-key', '/home/ubuntu/cephtest/ceph.pub', '--mon-ip', '172.21.15.28', '--skip-admin-label'] 2024-09-16T08:05:01.687 INFO:teuthology.orchestra.run.smithi028.stderr:/bin/podman: 5.2.2 2024-09-16T08:05:01.690 INFO:teuthology.orchestra.run.smithi028.stderr:Verifying podman|docker is present... 2024-09-16T08:05:01.711 INFO:teuthology.orchestra.run.smithi028.stderr:/bin/podman: 5.2.2 2024-09-16T08:05:01.713 INFO:teuthology.orchestra.run.smithi028.stderr:Verifying lvm2 is present... 2024-09-16T08:05:01.713 INFO:teuthology.orchestra.run.smithi028.stderr:Verifying time synchronization is in place... 2024-09-16T08:05:01.720 INFO:teuthology.orchestra.run.smithi028.stderr:systemctl: Failed to get unit file state for chrony.service: No such file or directory 2024-09-16T08:05:01.727 INFO:teuthology.orchestra.run.smithi028.stderr:systemctl: inactive 2024-09-16T08:05:01.734 INFO:teuthology.orchestra.run.smithi028.stderr:systemctl: enabled 2024-09-16T08:05:01.741 INFO:teuthology.orchestra.run.smithi028.stderr:systemctl: active 2024-09-16T08:05:01.741 INFO:teuthology.orchestra.run.smithi028.stderr:Unit chronyd.service is enabled and running 2024-09-16T08:05:01.741 INFO:teuthology.orchestra.run.smithi028.stderr:Repeating the final host check... 2024-09-16T08:05:01.762 INFO:teuthology.orchestra.run.smithi028.stderr:/bin/podman: 5.2.2 2024-09-16T08:05:01.764 INFO:teuthology.orchestra.run.smithi028.stderr:podman (/bin/podman) version 5.2.2 is present 2024-09-16T08:05:01.764 INFO:teuthology.orchestra.run.smithi028.stderr:systemctl is present 2024-09-16T08:05:01.764 INFO:teuthology.orchestra.run.smithi028.stderr:lvcreate is present 2024-09-16T08:05:01.770 INFO:teuthology.orchestra.run.smithi028.stderr:systemctl: Failed to get unit file state for chrony.service: No such file or directory 2024-09-16T08:05:01.777 INFO:teuthology.orchestra.run.smithi028.stderr:systemctl: inactive 2024-09-16T08:05:01.784 INFO:teuthology.orchestra.run.smithi028.stderr:systemctl: enabled 2024-09-16T08:05:01.790 INFO:teuthology.orchestra.run.smithi028.stderr:systemctl: active 2024-09-16T08:05:01.791 INFO:teuthology.orchestra.run.smithi028.stderr:Unit chronyd.service is enabled and running 2024-09-16T08:05:01.792 INFO:teuthology.orchestra.run.smithi028.stderr:Host looks OK 2024-09-16T08:05:01.792 INFO:teuthology.orchestra.run.smithi028.stderr:Cluster fsid: 5622b2fe-7402-11ef-bceb-c7b262605968 2024-09-16T08:05:01.792 INFO:teuthology.orchestra.run.smithi028.stderr:Acquiring lock 140460032075664 on /run/cephadm/5622b2fe-7402-11ef-bceb-c7b262605968.lock 2024-09-16T08:05:01.792 INFO:teuthology.orchestra.run.smithi028.stderr:Lock 140460032075664 acquired on /run/cephadm/5622b2fe-7402-11ef-bceb-c7b262605968.lock 2024-09-16T08:05:01.792 INFO:teuthology.orchestra.run.smithi028.stderr:Verifying IP 172.21.15.28 port 3300 ... 2024-09-16T08:05:01.793 INFO:teuthology.orchestra.run.smithi028.stderr:Verifying IP 172.21.15.28 port 6789 ... 2024-09-16T08:05:01.793 INFO:teuthology.orchestra.run.smithi028.stderr:Base mon IP is 172.21.15.28, final addrv is [v2:172.21.15.28:3300,v1:172.21.15.28:6789] 2024-09-16T08:05:01.795 INFO:teuthology.orchestra.run.smithi028.stderr:/sbin/ip: default via 172.21.15.254 dev ens1f0 proto dhcp src 172.21.15.28 metric 100 2024-09-16T08:05:01.795 INFO:teuthology.orchestra.run.smithi028.stderr:/sbin/ip: 172.21.0.0/20 dev ens1f0 proto kernel scope link src 172.21.15.28 metric 100 2024-09-16T08:05:01.798 INFO:teuthology.orchestra.run.smithi028.stderr:/sbin/ip: ::1 dev lo proto kernel metric 256 pref medium 2024-09-16T08:05:01.798 INFO:teuthology.orchestra.run.smithi028.stderr:/sbin/ip: fe80::/64 dev ens1f0 proto kernel metric 256 pref medium 2024-09-16T08:05:01.798 INFO:teuthology.orchestra.run.smithi028.stderr:/sbin/ip: default via fe80::327c:5e00:6487:71e0 dev ens1f0 proto ra metric 1024 expires 1767sec hoplimit 64 pref medium 2024-09-16T08:05:01.801 INFO:teuthology.orchestra.run.smithi028.stderr:/sbin/ip: 1: lo: mtu 65536 state UNKNOWN qlen 1000 2024-09-16T08:05:01.801 INFO:teuthology.orchestra.run.smithi028.stderr:/sbin/ip: inet6 ::1/128 scope host 2024-09-16T08:05:01.801 INFO:teuthology.orchestra.run.smithi028.stderr:/sbin/ip: valid_lft forever preferred_lft forever 2024-09-16T08:05:01.801 INFO:teuthology.orchestra.run.smithi028.stderr:/sbin/ip: 4: ens1f0: mtu 1500 state UP qlen 1000 2024-09-16T08:05:01.801 INFO:teuthology.orchestra.run.smithi028.stderr:/sbin/ip: inet6 fe80::ae1f:6bff:fea5:ca3c/64 scope link 2024-09-16T08:05:01.801 INFO:teuthology.orchestra.run.smithi028.stderr:/sbin/ip: valid_lft forever preferred_lft forever 2024-09-16T08:05:01.801 INFO:teuthology.orchestra.run.smithi028.stderr:Mon IP `172.21.15.28` is in CIDR network `172.21.0.0/20` 2024-09-16T08:05:01.802 INFO:teuthology.orchestra.run.smithi028.stderr:- internal network (--cluster-network) has not been provided, OSD replication will default to the public_network 2024-09-16T08:05:01.802 INFO:teuthology.orchestra.run.smithi028.stderr:Pulling container image quay.io/ceph/ceph:v17.2.0... 2024-09-16T08:05:01.906 INFO:teuthology.orchestra.run.smithi028.stderr:/bin/podman: Trying to pull quay.io/ceph/ceph:v17.2.0... 2024-09-16T08:05:02.264 INFO:teuthology.orchestra.run.smithi028.stderr:/bin/podman: Getting image source signatures 2024-09-16T08:05:02.264 INFO:teuthology.orchestra.run.smithi028.stderr:/bin/podman: Copying blob sha256:33ca8fff7868c4dc0c11e09bca97c720eb9cfbab7221216754367dd8de70388a 2024-09-16T08:05:02.264 INFO:teuthology.orchestra.run.smithi028.stderr:/bin/podman: Copying blob sha256:89b4a75bc2d8500f15463747507c9623df43886c134463e7f0527e70900e7a7b 2024-09-16T08:05:02.264 INFO:teuthology.orchestra.run.smithi028.stderr:/bin/podman: Copying blob sha256:c32ab78b488d0c72f64eded765c0cf6b5bf2c75dab66cb62a9d367fa6ec42513 2024-09-16T08:05:02.264 INFO:teuthology.orchestra.run.smithi028.stderr:/bin/podman: Copying blob sha256:a70843738bb77e1ab9c1f85969ebdfa55f178e746be081d1cb4f94011f69eb7c 2024-09-16T08:05:02.265 INFO:teuthology.orchestra.run.smithi028.stderr:/bin/podman: Copying blob sha256:599d07cb321ff0a3c82224e1138fc685793fa69b93ed5780415751a5f7e4b8c2 2024-09-16T08:05:02.265 INFO:teuthology.orchestra.run.smithi028.stderr:/bin/podman: Copying config sha256:e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 2024-09-16T08:05:02.433 INFO:teuthology.orchestra.run.smithi028.stderr:/bin/podman: Writing manifest to image destination 2024-09-16T08:05:02.602 INFO:teuthology.orchestra.run.smithi028.stderr:/bin/podman: e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 2024-09-16T08:05:02.990 INFO:teuthology.orchestra.run.smithi028.stderr:ceph: ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable) 2024-09-16T08:05:05.018 INFO:teuthology.orchestra.run.smithi028.stderr:Ceph version: ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable) 2024-09-16T08:05:05.018 INFO:teuthology.orchestra.run.smithi028.stderr:Extracting ceph user uid/gid from container image... 2024-09-16T08:05:08.822 INFO:teuthology.orchestra.run.smithi028.stderr:stat: 167 167 2024-09-16T08:05:13.090 INFO:teuthology.orchestra.run.smithi028.stderr:Creating initial keys... 2024-09-16T08:05:13.488 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph-authtool: AQC55udmUMu4GhAARlvjEFCtv388nentXZnFiw== 2024-09-16T08:05:14.385 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph-authtool: AQC65udm8e6gERAAYnd9vc36yDzJGYmEs0RIjg== 2024-09-16T08:05:15.366 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph-authtool: AQC75udmRe95ERAAP9XEmXrqJCSaWDeT18CnlA== 2024-09-16T08:05:15.822 INFO:teuthology.orchestra.run.smithi028.stderr:Creating initial monmap... 2024-09-16T08:05:16.204 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/monmaptool: /usr/bin/monmaptool: monmap file /tmp/monmap 2024-09-16T08:05:16.204 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/monmaptool: setting min_mon_release = octopus 2024-09-16T08:05:16.205 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/monmaptool: /usr/bin/monmaptool: set fsid to 5622b2fe-7402-11ef-bceb-c7b262605968 2024-09-16T08:05:16.205 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/monmaptool: /usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2024-09-16T08:05:16.727 INFO:teuthology.orchestra.run.smithi028.stderr:monmaptool for smithi028 [v2:172.21.15.28:3300,v1:172.21.15.28:6789] on /usr/bin/monmaptool: monmap file /tmp/monmap 2024-09-16T08:05:16.727 INFO:teuthology.orchestra.run.smithi028.stderr:setting min_mon_release = octopus 2024-09-16T08:05:16.727 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/monmaptool: set fsid to 5622b2fe-7402-11ef-bceb-c7b262605968 2024-09-16T08:05:16.727 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2024-09-16T08:05:16.727 INFO:teuthology.orchestra.run.smithi028.stderr: 2024-09-16T08:05:16.727 INFO:teuthology.orchestra.run.smithi028.stderr:Creating mon... 2024-09-16T08:05:18.889 INFO:teuthology.orchestra.run.smithi028.stderr:create mon.smithi028 on 2024-09-16T08:05:19.105 INFO:teuthology.orchestra.run.smithi028.stderr:systemctl: Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target → /etc/systemd/system/ceph.target. 2024-09-16T08:05:19.265 INFO:teuthology.orchestra.run.smithi028.stderr:systemctl: Created symlink /etc/systemd/system/multi-user.target.wants/ceph-5622b2fe-7402-11ef-bceb-c7b262605968.target → /etc/systemd/system/ceph-5622b2fe-7402-11ef-bceb-c7b262605968.target. 2024-09-16T08:05:19.265 INFO:teuthology.orchestra.run.smithi028.stderr:systemctl: Created symlink /etc/systemd/system/ceph.target.wants/ceph-5622b2fe-7402-11ef-bceb-c7b262605968.target → /etc/systemd/system/ceph-5622b2fe-7402-11ef-bceb-c7b262605968.target. 2024-09-16T08:05:19.589 INFO:teuthology.orchestra.run.smithi028.stderr:systemctl: Failed to reset failed state of unit ceph-5622b2fe-7402-11ef-bceb-c7b262605968@mon.smithi028.service: Unit ceph-5622b2fe-7402-11ef-bceb-c7b262605968@mon.smithi028.service not loaded. 2024-09-16T08:05:19.597 INFO:teuthology.orchestra.run.smithi028.stderr:systemctl: Created symlink /etc/systemd/system/ceph-5622b2fe-7402-11ef-bceb-c7b262605968.target.wants/ceph-5622b2fe-7402-11ef-bceb-c7b262605968@mon.smithi028.service → /etc/systemd/system/ceph-5622b2fe-7402-11ef-bceb-c7b262605968@.service. 2024-09-16T08:05:20.255 INFO:teuthology.orchestra.run.smithi028.stderr:systemctl: disabled 2024-09-16T08:05:20.261 INFO:teuthology.orchestra.run.smithi028.stderr:systemctl: inactive 2024-09-16T08:05:20.262 INFO:teuthology.orchestra.run.smithi028.stderr:firewalld.service is not enabled 2024-09-16T08:05:20.262 INFO:teuthology.orchestra.run.smithi028.stderr:Not possible to enable service . firewalld.service is not available 2024-09-16T08:05:20.263 INFO:teuthology.orchestra.run.smithi028.stderr:Waiting for mon to start... 2024-09-16T08:05:20.263 INFO:teuthology.orchestra.run.smithi028.stderr:Waiting for mon... 2024-09-16T08:05:20.376 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:20 smithi028 systemd[1]: Started Ceph mon.smithi028 for 5622b2fe-7402-11ef-bceb-c7b262605968. 2024-09-16T08:05:20.758 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: cluster: 2024-09-16T08:05:20.758 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: id: 5622b2fe-7402-11ef-bceb-c7b262605968 2024-09-16T08:05:20.758 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: health: HEALTH_OK 2024-09-16T08:05:20.758 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: 2024-09-16T08:05:20.758 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: services: 2024-09-16T08:05:20.758 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: mon: 1 daemons, quorum smithi028 (age 0.533403s) 2024-09-16T08:05:20.758 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: mgr: no daemons active 2024-09-16T08:05:20.758 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: osd: 0 osds: 0 up, 0 in 2024-09-16T08:05:20.759 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: 2024-09-16T08:05:20.759 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: data: 2024-09-16T08:05:20.759 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: pools: 0 pools, 0 pgs 2024-09-16T08:05:20.759 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: objects: 0 objects, 0 B 2024-09-16T08:05:20.759 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: usage: 0 B used, 0 B / 0 B avail 2024-09-16T08:05:20.759 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: pgs: 2024-09-16T08:05:20.759 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: 2024-09-16T08:05:21.405 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:21 smithi028 ceph-mon[23405]: mon.smithi028 is new leader, mons smithi028 in quorum (ranks 0) 2024-09-16T08:05:21.406 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:21 smithi028 ceph-mon[23405]: monmap e1: 1 mons at {smithi028=[v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0]} 2024-09-16T08:05:21.406 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:21 smithi028 ceph-mon[23405]: fsmap 2024-09-16T08:05:21.406 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:21 smithi028 ceph-mon[23405]: osdmap e1: 0 total, 0 up, 0 in 2024-09-16T08:05:21.406 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:21 smithi028 ceph-mon[23405]: mgrmap e1: no daemons active 2024-09-16T08:05:21.406 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:21 smithi028 ceph-mon[23405]: from='client.? 172.21.15.28:0/3787478569' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-09-16T08:05:21.408 INFO:teuthology.orchestra.run.smithi028.stderr:mon is available 2024-09-16T08:05:21.408 INFO:teuthology.orchestra.run.smithi028.stderr:Assimilating anything we can from ceph.conf... 2024-09-16T08:05:21.922 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: 2024-09-16T08:05:21.922 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: [global] 2024-09-16T08:05:21.922 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: fsid = 5622b2fe-7402-11ef-bceb-c7b262605968 2024-09-16T08:05:21.923 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: mon_host = [v2:172.21.15.28:3300,v1:172.21.15.28:6789] 2024-09-16T08:05:21.923 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: mon_osd_allow_pg_remap = true 2024-09-16T08:05:21.923 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: mon_osd_allow_primary_affinity = true 2024-09-16T08:05:21.923 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: mon_warn_on_no_sortbitwise = false 2024-09-16T08:05:21.923 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: osd_crush_chooseleaf_type = 0 2024-09-16T08:05:21.923 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: 2024-09-16T08:05:21.923 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: [mgr] 2024-09-16T08:05:21.923 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: mgr/telemetry/nag = false 2024-09-16T08:05:21.923 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: 2024-09-16T08:05:21.923 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: [osd] 2024-09-16T08:05:21.923 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: osd_map_max_advance = 10 2024-09-16T08:05:21.923 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: osd_sloppy_crc = true 2024-09-16T08:05:22.486 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:22 smithi028 ceph-mon[23405]: from='client.? 172.21.15.28:0/2472829431' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2024-09-16T08:05:22.486 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:22 smithi028 ceph-mon[23405]: from='client.? 172.21.15.28:0/2472829431' entity='client.admin' cmd='[{"prefix": "config assimilate-conf"}]': finished 2024-09-16T08:05:22.488 INFO:teuthology.orchestra.run.smithi028.stderr:Generating new minimal ceph.conf... 2024-09-16T08:05:23.533 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:23 smithi028 ceph-mon[23405]: from='client.? 172.21.15.28:0/3431688179' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T08:05:23.536 INFO:teuthology.orchestra.run.smithi028.stderr:Restarting the monitor... 2024-09-16T08:05:23.947 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:23 smithi028 systemd[1]: Stopping Ceph mon.smithi028 for 5622b2fe-7402-11ef-bceb-c7b262605968... 2024-09-16T08:05:23.947 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:23 smithi028 bash[23623]: Error: no container with name or ID "ceph-5622b2fe-7402-11ef-bceb-c7b262605968-mon.smithi028" found: no such container 2024-09-16T08:05:23.947 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:23 smithi028 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-mon-smithi028[23401]: 2024-09-16T08:05:23.679+0000 7fe4d16f3700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.smithi028 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-09-16T08:05:23.947 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:23 smithi028 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-mon-smithi028[23401]: 2024-09-16T08:05:23.679+0000 7fe4d16f3700 -1 mon.smithi028@0(leader) e1 *** Got Signal Terminated *** 2024-09-16T08:05:23.947 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:23 smithi028 podman[23634]: 2024-09-16 08:05:23.751242044 +0000 UTC m=+0.152862831 container died 8b3bd68f27330f61417f852091a3fae4c749097d124c30f6a65a942bfb3e8c8b (image=quay.io/ceph/ceph:v17.2.0, name=ceph-5622b2fe-7402-11ef-bceb-c7b262605968-mon-smithi028, CEPH_POINT_RELEASE=-17.2.0, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_CLEAN=True, name=centos-stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.buildah.version=1.19.8, vcs-type=git, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.component=centos-stream-container, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, release=754, GIT_BRANCH=HEAD, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, architecture=x86_64, RELEASE=HEAD, io.openshift.expose-services=, io.openshift.tags=base centos centos-stream, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.k8s.display-name=CentOS Stream 8, ceph=True, maintainer=Guillaume Abrioux , build-date=2022-05-03T08:36:31.336870, version=8, GIT_REPO=https://github.com/ceph/ceph-container.git, distribution-scope=public, vendor=Red Hat, Inc., io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly.) 2024-09-16T08:05:24.446 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:24 smithi028 podman[23634]: 2024-09-16 08:05:24.27048236 +0000 UTC m=+0.672103134 container cleanup 8b3bd68f27330f61417f852091a3fae4c749097d124c30f6a65a942bfb3e8c8b (image=quay.io/ceph/ceph:v17.2.0, name=ceph-5622b2fe-7402-11ef-bceb-c7b262605968-mon-smithi028, build-date=2022-05-03T08:36:31.336870, distribution-scope=public, maintainer=Guillaume Abrioux , name=centos-stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_CLEAN=True, CEPH_POINT_RELEASE=-17.2.0, architecture=x86_64, io.openshift.expose-services=, io.buildah.version=1.19.8, io.k8s.display-name=CentOS Stream 8, release=754, vendor=Red Hat, Inc., com.redhat.component=centos-stream-container, RELEASE=HEAD, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, ceph=True, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, version=8, GIT_BRANCH=HEAD, io.openshift.tags=base centos centos-stream, GIT_REPO=https://github.com/ceph/ceph-container.git, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-type=git) 2024-09-16T08:05:24.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:24 smithi028 bash[23634]: ceph-5622b2fe-7402-11ef-bceb-c7b262605968-mon-smithi028 2024-09-16T08:05:24.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:24 smithi028 bash[23659]: Error: no container with name or ID "ceph-5622b2fe-7402-11ef-bceb-c7b262605968-mon.smithi028" found: no such container 2024-09-16T08:05:24.712 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:24 smithi028 podman[23646]: 2024-09-16 08:05:24.454689316 +0000 UTC m=+0.705589461 container remove 8b3bd68f27330f61417f852091a3fae4c749097d124c30f6a65a942bfb3e8c8b (image=quay.io/ceph/ceph:v17.2.0, name=ceph-5622b2fe-7402-11ef-bceb-c7b262605968-mon-smithi028, vendor=Red Hat, Inc., CEPH_POINT_RELEASE=-17.2.0, maintainer=Guillaume Abrioux , url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, architecture=x86_64, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vcs-type=git, RELEASE=HEAD, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.k8s.display-name=CentOS Stream 8, io.openshift.expose-services=, version=8, build-date=2022-05-03T08:36:31.336870, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.buildah.version=1.19.8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.openshift.tags=base centos centos-stream, GIT_CLEAN=True, name=centos-stream, com.redhat.component=centos-stream-container, distribution-scope=public, GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_BRANCH=HEAD, ceph=True, release=754) 2024-09-16T08:05:24.713 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:24 smithi028 systemd[1]: ceph-5622b2fe-7402-11ef-bceb-c7b262605968@mon.smithi028.service: Deactivated successfully. 2024-09-16T08:05:24.713 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:24 smithi028 systemd[1]: Stopped Ceph mon.smithi028 for 5622b2fe-7402-11ef-bceb-c7b262605968. 2024-09-16T08:05:24.713 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:24 smithi028 systemd[1]: Starting Ceph mon.smithi028 for 5622b2fe-7402-11ef-bceb-c7b262605968... 2024-09-16T08:05:24.713 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:24 smithi028 podman[23726]: 2024-09-16 08:05:24.600444495 +0000 UTC m=+0.017064239 image pull e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 quay.io/ceph/ceph:v17.2.0 2024-09-16T08:05:24.713 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:24 smithi028 podman[23726]: 2024-09-16 08:05:24.713652978 +0000 UTC m=+0.130272720 container create a2f688697052cb0def56c126ac145a22be239a549f59f8d80465942185c8d025 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-5622b2fe-7402-11ef-bceb-c7b262605968-mon-smithi028, CEPH_POINT_RELEASE=-17.2.0, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., architecture=x86_64, io.openshift.expose-services=, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, name=centos-stream, GIT_BRANCH=HEAD, vcs-type=git, build-date=2022-05-03T08:36:31.336870, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.buildah.version=1.19.8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, RELEASE=HEAD, distribution-scope=public, version=8, io.openshift.tags=base centos centos-stream, GIT_REPO=https://github.com/ceph/ceph-container.git, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_CLEAN=True, com.redhat.component=centos-stream-container, ceph=True, vendor=Red Hat, Inc., maintainer=Guillaume Abrioux , com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.k8s.display-name=CentOS Stream 8, release=754) 2024-09-16T08:05:25.068 INFO:teuthology.orchestra.run.smithi028.stderr:Setting mon public_network to 172.21.0.0/20 2024-09-16T08:05:25.259 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:24 smithi028 podman[23726]: 2024-09-16 08:05:24.965787417 +0000 UTC m=+0.382407164 container init a2f688697052cb0def56c126ac145a22be239a549f59f8d80465942185c8d025 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-5622b2fe-7402-11ef-bceb-c7b262605968-mon-smithi028, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, RELEASE=HEAD, architecture=x86_64, ceph=True, GIT_CLEAN=True, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.openshift.tags=base centos centos-stream, GIT_REPO=https://github.com/ceph/ceph-container.git, build-date=2022-05-03T08:36:31.336870, release=754, version=8, vendor=Red Hat, Inc., maintainer=Guillaume Abrioux , io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.buildah.version=1.19.8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_BRANCH=HEAD, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.k8s.display-name=CentOS Stream 8, vcs-type=git, name=centos-stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, CEPH_POINT_RELEASE=-17.2.0, io.openshift.expose-services=, com.redhat.component=centos-stream-container, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, distribution-scope=public) 2024-09-16T08:05:25.259 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:24 smithi028 podman[23726]: 2024-09-16 08:05:24.969278738 +0000 UTC m=+0.385898475 container start a2f688697052cb0def56c126ac145a22be239a549f59f8d80465942185c8d025 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-5622b2fe-7402-11ef-bceb-c7b262605968-mon-smithi028, release=754, GIT_BRANCH=HEAD, io.buildah.version=1.19.8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, CEPH_POINT_RELEASE=-17.2.0, io.k8s.display-name=CentOS Stream 8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, ceph=True, architecture=x86_64, version=8, build-date=2022-05-03T08:36:31.336870, maintainer=Guillaume Abrioux , summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, distribution-scope=public, RELEASE=HEAD, io.openshift.expose-services=, vendor=Red Hat, Inc., GIT_REPO=https://github.com/ceph/ceph-container.git, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.tags=base centos centos-stream, com.redhat.component=centos-stream-container, GIT_CLEAN=True, name=centos-stream, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, vcs-type=git, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com) 2024-09-16T08:05:25.259 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:24 smithi028 ceph-mon[23744]: set uid:gid to 167:167 (ceph:ceph) 2024-09-16T08:05:25.259 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:24 smithi028 ceph-mon[23744]: ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable), process ceph-mon, pid 2 2024-09-16T08:05:25.259 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:24 smithi028 ceph-mon[23744]: pidfile_write: ignore empty --pid-file 2024-09-16T08:05:25.260 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:24 smithi028 ceph-mon[23744]: load: jerasure load: lrc 2024-09-16T08:05:25.260 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:24 smithi028 ceph-mon[23744]: rocksdb: RocksDB version: 6.15.5 2024-09-16T08:05:25.260 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:24 smithi028 ceph-mon[23744]: rocksdb: Git sha rocksdb_build_git_sha:@0@ 2024-09-16T08:05:25.260 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:24 smithi028 ceph-mon[23744]: rocksdb: Compile date Apr 18 2022 2024-09-16T08:05:25.260 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:24 smithi028 ceph-mon[23744]: rocksdb: DB SUMMARY 2024-09-16T08:05:25.260 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:24 smithi028 ceph-mon[23744]: rocksdb: DB Session ID: AJVSL2TIULFTW0OEMD94 2024-09-16T08:05:25.260 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:24 smithi028 ceph-mon[23744]: rocksdb: CURRENT file: CURRENT 2024-09-16T08:05:25.260 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:24 smithi028 ceph-mon[23744]: rocksdb: IDENTITY file: IDENTITY 2024-09-16T08:05:25.260 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:24 smithi028 ceph-mon[23744]: rocksdb: MANIFEST file: MANIFEST-000009 size: 131 Bytes 2024-09-16T08:05:25.260 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:24 smithi028 ceph-mon[23744]: rocksdb: SST files in /var/lib/ceph/mon/ceph-smithi028/store.db dir, Total Num: 1, files: 000008.sst 2024-09-16T08:05:25.261 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:24 smithi028 ceph-mon[23744]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-smithi028/store.db: 000010.log size: 82713 ; 2024-09-16T08:05:25.261 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:24 smithi028 ceph-mon[23744]: rocksdb: Options.error_if_exists: 0 2024-09-16T08:05:25.261 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:24 smithi028 ceph-mon[23744]: rocksdb: Options.create_if_missing: 0 2024-09-16T08:05:25.261 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:24 smithi028 ceph-mon[23744]: rocksdb: Options.paranoid_checks: 1 2024-09-16T08:05:25.261 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:24 smithi028 ceph-mon[23744]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-09-16T08:05:25.261 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:24 smithi028 ceph-mon[23744]: rocksdb: Options.env: 0x55a277ac6860 2024-09-16T08:05:25.261 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:24 smithi028 ceph-mon[23744]: rocksdb: Options.fs: Posix File System 2024-09-16T08:05:25.261 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:24 smithi028 ceph-mon[23744]: rocksdb: Options.info_log: 0x55a279fd9fa0 2024-09-16T08:05:25.261 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:24 smithi028 ceph-mon[23744]: rocksdb: Options.max_file_opening_threads: 16 2024-09-16T08:05:25.261 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:24 smithi028 ceph-mon[23744]: rocksdb: Options.statistics: (nil) 2024-09-16T08:05:25.261 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:24 smithi028 ceph-mon[23744]: rocksdb: Options.use_fsync: 0 2024-09-16T08:05:25.261 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:24 smithi028 ceph-mon[23744]: rocksdb: Options.max_log_file_size: 0 2024-09-16T08:05:25.262 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:24 smithi028 ceph-mon[23744]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-09-16T08:05:25.262 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:24 smithi028 ceph-mon[23744]: rocksdb: Options.log_file_time_to_roll: 0 2024-09-16T08:05:25.262 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:24 smithi028 ceph-mon[23744]: rocksdb: Options.keep_log_file_num: 1000 2024-09-16T08:05:25.262 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:24 smithi028 ceph-mon[23744]: rocksdb: Options.recycle_log_file_num: 0 2024-09-16T08:05:25.262 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:24 smithi028 ceph-mon[23744]: rocksdb: Options.allow_fallocate: 1 2024-09-16T08:05:25.262 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:24 smithi028 ceph-mon[23744]: rocksdb: Options.allow_mmap_reads: 0 2024-09-16T08:05:25.262 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:24 smithi028 ceph-mon[23744]: rocksdb: Options.allow_mmap_writes: 0 2024-09-16T08:05:25.262 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:24 smithi028 ceph-mon[23744]: rocksdb: Options.use_direct_reads: 0 2024-09-16T08:05:25.262 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:24 smithi028 ceph-mon[23744]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-09-16T08:05:25.262 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:24 smithi028 ceph-mon[23744]: rocksdb: Options.create_missing_column_families: 0 2024-09-16T08:05:25.262 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:24 smithi028 ceph-mon[23744]: rocksdb: Options.db_log_dir: 2024-09-16T08:05:25.262 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:24 smithi028 ceph-mon[23744]: rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-smithi028/store.db 2024-09-16T08:05:25.262 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:24 smithi028 ceph-mon[23744]: rocksdb: Options.table_cache_numshardbits: 6 2024-09-16T08:05:25.262 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:24 smithi028 ceph-mon[23744]: rocksdb: Options.WAL_ttl_seconds: 0 2024-09-16T08:05:25.263 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:24 smithi028 ceph-mon[23744]: rocksdb: Options.WAL_size_limit_MB: 0 2024-09-16T08:05:25.263 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:24 smithi028 ceph-mon[23744]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-09-16T08:05:25.263 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:24 smithi028 ceph-mon[23744]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-09-16T08:05:25.263 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:24 smithi028 ceph-mon[23744]: rocksdb: Options.is_fd_close_on_exec: 1 2024-09-16T08:05:25.263 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:24 smithi028 ceph-mon[23744]: rocksdb: Options.advise_random_on_open: 1 2024-09-16T08:05:25.264 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:24 smithi028 ceph-mon[23744]: rocksdb: Options.db_write_buffer_size: 0 2024-09-16T08:05:25.264 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:24 smithi028 ceph-mon[23744]: rocksdb: Options.write_buffer_manager: 0x55a27a0ca330 2024-09-16T08:05:25.264 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:24 smithi028 ceph-mon[23744]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-09-16T08:05:25.264 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:24 smithi028 ceph-mon[23744]: rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2024-09-16T08:05:25.264 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:24 smithi028 ceph-mon[23744]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-09-16T08:05:25.264 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:24 smithi028 ceph-mon[23744]: rocksdb: Options.use_adaptive_mutex: 0 2024-09-16T08:05:25.264 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:24 smithi028 ceph-mon[23744]: rocksdb: Options.rate_limiter: (nil) 2024-09-16T08:05:25.264 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:24 smithi028 ceph-mon[23744]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-09-16T08:05:25.264 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:24 smithi028 ceph-mon[23744]: rocksdb: Options.wal_recovery_mode: 2 2024-09-16T08:05:25.265 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:24 smithi028 ceph-mon[23744]: rocksdb: Options.enable_thread_tracking: 0 2024-09-16T08:05:25.265 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:24 smithi028 ceph-mon[23744]: rocksdb: Options.enable_pipelined_write: 0 2024-09-16T08:05:25.265 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:24 smithi028 ceph-mon[23744]: rocksdb: Options.unordered_write: 0 2024-09-16T08:05:25.265 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:24 smithi028 ceph-mon[23744]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-09-16T08:05:25.265 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:24 smithi028 ceph-mon[23744]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-09-16T08:05:25.265 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:24 smithi028 ceph-mon[23744]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-09-16T08:05:25.265 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:24 smithi028 ceph-mon[23744]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-09-16T08:05:25.265 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:24 smithi028 ceph-mon[23744]: rocksdb: Options.row_cache: None 2024-09-16T08:05:25.265 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:24 smithi028 ceph-mon[23744]: rocksdb: Options.wal_filter: None 2024-09-16T08:05:25.265 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:24 smithi028 ceph-mon[23744]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-09-16T08:05:25.265 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:24 smithi028 ceph-mon[23744]: rocksdb: Options.allow_ingest_behind: 0 2024-09-16T08:05:25.265 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:24 smithi028 ceph-mon[23744]: rocksdb: Options.preserve_deletes: 0 2024-09-16T08:05:25.265 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:24 smithi028 ceph-mon[23744]: rocksdb: Options.two_write_queues: 0 2024-09-16T08:05:25.265 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:24 smithi028 ceph-mon[23744]: rocksdb: Options.manual_wal_flush: 0 2024-09-16T08:05:25.265 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:24 smithi028 ceph-mon[23744]: rocksdb: Options.atomic_flush: 0 2024-09-16T08:05:25.266 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:24 smithi028 ceph-mon[23744]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-09-16T08:05:25.266 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:24 smithi028 ceph-mon[23744]: rocksdb: Options.persist_stats_to_disk: 0 2024-09-16T08:05:25.266 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:24 smithi028 ceph-mon[23744]: rocksdb: Options.write_dbid_to_manifest: 0 2024-09-16T08:05:25.266 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:24 smithi028 ceph-mon[23744]: rocksdb: Options.log_readahead_size: 0 2024-09-16T08:05:25.266 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:24 smithi028 ceph-mon[23744]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-09-16T08:05:25.266 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:24 smithi028 ceph-mon[23744]: rocksdb: Options.best_efforts_recovery: 0 2024-09-16T08:05:25.266 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:24 smithi028 ceph-mon[23744]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-09-16T08:05:25.266 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:24 smithi028 ceph-mon[23744]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-09-16T08:05:25.266 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:24 smithi028 ceph-mon[23744]: rocksdb: Options.allow_data_in_errors: 0 2024-09-16T08:05:25.266 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:24 smithi028 ceph-mon[23744]: rocksdb: Options.db_host_id: __hostname__ 2024-09-16T08:05:25.266 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:24 smithi028 ceph-mon[23744]: rocksdb: Options.max_background_jobs: 2 2024-09-16T08:05:25.266 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:24 smithi028 ceph-mon[23744]: rocksdb: Options.max_background_compactions: -1 2024-09-16T08:05:25.266 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:24 smithi028 ceph-mon[23744]: rocksdb: Options.max_subcompactions: 1 2024-09-16T08:05:25.267 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:24 smithi028 ceph-mon[23744]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-09-16T08:05:25.267 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:24 smithi028 ceph-mon[23744]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-09-16T08:05:25.267 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:24 smithi028 ceph-mon[23744]: rocksdb: Options.delayed_write_rate : 16777216 2024-09-16T08:05:25.267 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:24 smithi028 ceph-mon[23744]: rocksdb: Options.max_total_wal_size: 0 2024-09-16T08:05:25.267 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:24 smithi028 ceph-mon[23744]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-09-16T08:05:25.267 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:24 smithi028 ceph-mon[23744]: rocksdb: Options.stats_dump_period_sec: 600 2024-09-16T08:05:25.267 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:24 smithi028 ceph-mon[23744]: rocksdb: Options.stats_persist_period_sec: 600 2024-09-16T08:05:25.267 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:24 smithi028 ceph-mon[23744]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-09-16T08:05:25.267 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:24 smithi028 ceph-mon[23744]: rocksdb: Options.max_open_files: -1 2024-09-16T08:05:25.267 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:24 smithi028 ceph-mon[23744]: rocksdb: Options.bytes_per_sync: 0 2024-09-16T08:05:25.267 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:24 smithi028 ceph-mon[23744]: rocksdb: Options.wal_bytes_per_sync: 0 2024-09-16T08:05:25.267 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:24 smithi028 ceph-mon[23744]: rocksdb: Options.strict_bytes_per_sync: 0 2024-09-16T08:05:25.267 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:24 smithi028 ceph-mon[23744]: rocksdb: Options.compaction_readahead_size: 0 2024-09-16T08:05:25.267 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:24 smithi028 ceph-mon[23744]: rocksdb: Options.max_background_flushes: -1 2024-09-16T08:05:25.268 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:24 smithi028 ceph-mon[23744]: rocksdb: Compression algorithms supported: 2024-09-16T08:05:25.268 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:24 smithi028 ceph-mon[23744]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-09-16T08:05:25.268 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:24 smithi028 ceph-mon[23744]: rocksdb: kZSTD supported: 0 2024-09-16T08:05:25.268 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:24 smithi028 ceph-mon[23744]: rocksdb: kXpressCompression supported: 0 2024-09-16T08:05:25.268 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:24 smithi028 ceph-mon[23744]: rocksdb: kLZ4HCCompression supported: 1 2024-09-16T08:05:25.268 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:24 smithi028 ceph-mon[23744]: rocksdb: kLZ4Compression supported: 1 2024-09-16T08:05:25.268 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:24 smithi028 ceph-mon[23744]: rocksdb: kBZip2Compression supported: 0 2024-09-16T08:05:25.268 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:24 smithi028 ceph-mon[23744]: rocksdb: kZlibCompression supported: 1 2024-09-16T08:05:25.268 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:24 smithi028 ceph-mon[23744]: rocksdb: kSnappyCompression supported: 1 2024-09-16T08:05:25.268 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:24 smithi028 ceph-mon[23744]: rocksdb: Fast CRC32 supported: Supported on x86 2024-09-16T08:05:25.268 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:24 smithi028 ceph-mon[23744]: rocksdb: [db/version_set.cc:4725] Recovering from manifest file: /var/lib/ceph/mon/ceph-smithi028/store.db/MANIFEST-000009 2024-09-16T08:05:25.268 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:24 smithi028 ceph-mon[23744]: rocksdb: [db/column_family.cc:597] --------------- Options for column family [default]: 2024-09-16T08:05:25.268 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:24 smithi028 ceph-mon[23744]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-09-16T08:05:25.268 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:24 smithi028 ceph-mon[23744]: rocksdb: Options.merge_operator: 2024-09-16T08:05:25.269 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:24 smithi028 ceph-mon[23744]: rocksdb: Options.compaction_filter: None 2024-09-16T08:05:25.269 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:24 smithi028 ceph-mon[23744]: rocksdb: Options.compaction_filter_factory: None 2024-09-16T08:05:25.269 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:24 smithi028 ceph-mon[23744]: rocksdb: Options.sst_partitioner_factory: None 2024-09-16T08:05:25.269 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:24 smithi028 ceph-mon[23744]: rocksdb: Options.memtable_factory: SkipListFactory 2024-09-16T08:05:25.269 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:24 smithi028 ceph-mon[23744]: rocksdb: Options.table_factory: BlockBasedTable 2024-09-16T08:05:25.269 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:24 smithi028 ceph-mon[23744]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55a279fa5d00) 2024-09-16T08:05:25.269 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout: cache_index_and_filter_blocks: 1 2024-09-16T08:05:25.269 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-09-16T08:05:25.269 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-09-16T08:05:25.269 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout: pin_top_level_index_and_filter: 1 2024-09-16T08:05:25.269 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout: index_type: 0 2024-09-16T08:05:25.269 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout: data_block_index_type: 0 2024-09-16T08:05:25.269 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout: index_shortening: 1 2024-09-16T08:05:25.269 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout: data_block_hash_table_util_ratio: 0.750000 2024-09-16T08:05:25.269 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout: hash_index_allow_collision: 1 2024-09-16T08:05:25.270 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout: checksum: 1 2024-09-16T08:05:25.270 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout: no_block_cache: 0 2024-09-16T08:05:25.270 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout: block_cache: 0x55a27a010170 2024-09-16T08:05:25.270 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout: block_cache_name: BinnedLRUCache 2024-09-16T08:05:25.270 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout: block_cache_options: 2024-09-16T08:05:25.270 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout: capacity : 536870912 2024-09-16T08:05:25.270 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout: num_shard_bits : 4 2024-09-16T08:05:25.270 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout: strict_capacity_limit : 0 2024-09-16T08:05:25.270 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout: high_pri_pool_ratio: 0.000 2024-09-16T08:05:25.270 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout: block_cache_compressed: (nil) 2024-09-16T08:05:25.270 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout: persistent_cache: (nil) 2024-09-16T08:05:25.270 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout: block_size: 4096 2024-09-16T08:05:25.270 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout: block_size_deviation: 10 2024-09-16T08:05:25.271 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout: block_restart_interval: 16 2024-09-16T08:05:25.271 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout: index_block_restart_interval: 1 2024-09-16T08:05:25.271 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout: metadata_block_size: 4096 2024-09-16T08:05:25.271 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout: partition_filters: 0 2024-09-16T08:05:25.271 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout: use_delta_encoding: 1 2024-09-16T08:05:25.271 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout: filter_policy: rocksdb.BuiltinBloomFilter 2024-09-16T08:05:25.271 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout: whole_key_filtering: 1 2024-09-16T08:05:25.271 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout: verify_compression: 0 2024-09-16T08:05:25.271 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout: read_amp_bytes_per_bit: 0 2024-09-16T08:05:25.271 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout: format_version: 4 2024-09-16T08:05:25.271 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout: enable_index_compression: 1 2024-09-16T08:05:25.272 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout: block_align: 0 2024-09-16T08:05:25.272 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:24 smithi028 ceph-mon[23744]: rocksdb: Options.write_buffer_size: 33554432 2024-09-16T08:05:25.272 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:24 smithi028 ceph-mon[23744]: rocksdb: Options.max_write_buffer_number: 2 2024-09-16T08:05:25.272 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:24 smithi028 ceph-mon[23744]: rocksdb: Options.compression: NoCompression 2024-09-16T08:05:25.272 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:24 smithi028 ceph-mon[23744]: rocksdb: Options.bottommost_compression: Disabled 2024-09-16T08:05:25.272 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:24 smithi028 ceph-mon[23744]: rocksdb: Options.prefix_extractor: nullptr 2024-09-16T08:05:25.272 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:24 smithi028 ceph-mon[23744]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-09-16T08:05:25.272 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:24 smithi028 ceph-mon[23744]: rocksdb: Options.num_levels: 7 2024-09-16T08:05:25.272 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:24 smithi028 ceph-mon[23744]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-09-16T08:05:25.272 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:24 smithi028 ceph-mon[23744]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-09-16T08:05:25.272 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:24 smithi028 ceph-mon[23744]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-09-16T08:05:25.272 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:24 smithi028 ceph-mon[23744]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-09-16T08:05:25.272 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:24 smithi028 ceph-mon[23744]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-09-16T08:05:25.272 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:24 smithi028 ceph-mon[23744]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-09-16T08:05:25.273 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:24 smithi028 ceph-mon[23744]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-09-16T08:05:25.273 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:24 smithi028 ceph-mon[23744]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-09-16T08:05:25.273 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:24 smithi028 ceph-mon[23744]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-09-16T08:05:25.273 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:24 smithi028 ceph-mon[23744]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-09-16T08:05:25.273 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:24 smithi028 ceph-mon[23744]: rocksdb: Options.compression_opts.window_bits: -14 2024-09-16T08:05:25.273 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:24 smithi028 ceph-mon[23744]: rocksdb: Options.compression_opts.level: 32767 2024-09-16T08:05:25.273 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:24 smithi028 ceph-mon[23744]: rocksdb: Options.compression_opts.strategy: 0 2024-09-16T08:05:25.273 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:24 smithi028 ceph-mon[23744]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-09-16T08:05:25.273 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:24 smithi028 ceph-mon[23744]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-09-16T08:05:25.273 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:24 smithi028 ceph-mon[23744]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-09-16T08:05:25.273 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:24 smithi028 ceph-mon[23744]: rocksdb: Options.compression_opts.enabled: false 2024-09-16T08:05:25.273 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:24 smithi028 ceph-mon[23744]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-09-16T08:05:25.273 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:24 smithi028 ceph-mon[23744]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-09-16T08:05:25.273 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:24 smithi028 ceph-mon[23744]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-09-16T08:05:25.273 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:24 smithi028 ceph-mon[23744]: rocksdb: Options.target_file_size_base: 67108864 2024-09-16T08:05:25.273 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:24 smithi028 ceph-mon[23744]: rocksdb: Options.target_file_size_multiplier: 1 2024-09-16T08:05:25.274 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:24 smithi028 ceph-mon[23744]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-09-16T08:05:25.274 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:24 smithi028 ceph-mon[23744]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-09-16T08:05:25.274 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:24 smithi028 ceph-mon[23744]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-09-16T08:05:25.274 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:24 smithi028 ceph-mon[23744]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-09-16T08:05:25.274 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:24 smithi028 ceph-mon[23744]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-09-16T08:05:25.274 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:24 smithi028 ceph-mon[23744]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-09-16T08:05:25.274 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:24 smithi028 ceph-mon[23744]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-09-16T08:05:25.274 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:24 smithi028 ceph-mon[23744]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-09-16T08:05:25.274 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:24 smithi028 ceph-mon[23744]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-09-16T08:05:25.274 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:24 smithi028 ceph-mon[23744]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-09-16T08:05:25.274 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:24 smithi028 ceph-mon[23744]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-09-16T08:05:25.274 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:24 smithi028 ceph-mon[23744]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-09-16T08:05:25.274 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:24 smithi028 ceph-mon[23744]: rocksdb: Options.arena_block_size: 4194304 2024-09-16T08:05:25.274 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:24 smithi028 ceph-mon[23744]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-09-16T08:05:25.274 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:24 smithi028 ceph-mon[23744]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-09-16T08:05:25.274 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:24 smithi028 ceph-mon[23744]: rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2024-09-16T08:05:25.275 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:24 smithi028 ceph-mon[23744]: rocksdb: Options.disable_auto_compactions: 0 2024-09-16T08:05:25.275 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:24 smithi028 ceph-mon[23744]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-09-16T08:05:25.275 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:24 smithi028 ceph-mon[23744]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-09-16T08:05:25.275 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:24 smithi028 ceph-mon[23744]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-09-16T08:05:25.275 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:24 smithi028 ceph-mon[23744]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-09-16T08:05:25.275 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:24 smithi028 ceph-mon[23744]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-09-16T08:05:25.275 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:24 smithi028 ceph-mon[23744]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-09-16T08:05:25.275 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:24 smithi028 ceph-mon[23744]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-09-16T08:05:25.275 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:24 smithi028 ceph-mon[23744]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-09-16T08:05:25.275 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:24 smithi028 ceph-mon[23744]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-09-16T08:05:25.275 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:24 smithi028 ceph-mon[23744]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-09-16T08:05:25.275 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:24 smithi028 ceph-mon[23744]: rocksdb: Options.table_properties_collectors: 2024-09-16T08:05:25.275 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:24 smithi028 ceph-mon[23744]: rocksdb: Options.inplace_update_support: 0 2024-09-16T08:05:25.275 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:24 smithi028 ceph-mon[23744]: rocksdb: Options.inplace_update_num_locks: 10000 2024-09-16T08:05:25.275 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:24 smithi028 ceph-mon[23744]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-09-16T08:05:25.276 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:24 smithi028 ceph-mon[23744]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-09-16T08:05:25.276 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:24 smithi028 ceph-mon[23744]: rocksdb: Options.memtable_huge_page_size: 0 2024-09-16T08:05:25.276 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:24 smithi028 ceph-mon[23744]: rocksdb: Options.bloom_locality: 0 2024-09-16T08:05:25.276 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:24 smithi028 ceph-mon[23744]: rocksdb: Options.max_successive_merges: 0 2024-09-16T08:05:25.276 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:24 smithi028 ceph-mon[23744]: rocksdb: Options.optimize_filters_for_hits: 0 2024-09-16T08:05:25.276 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:24 smithi028 ceph-mon[23744]: rocksdb: Options.paranoid_file_checks: 0 2024-09-16T08:05:25.276 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:24 smithi028 ceph-mon[23744]: rocksdb: Options.force_consistency_checks: 1 2024-09-16T08:05:25.276 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:24 smithi028 ceph-mon[23744]: rocksdb: Options.report_bg_io_stats: 0 2024-09-16T08:05:25.276 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:24 smithi028 ceph-mon[23744]: rocksdb: Options.ttl: 2592000 2024-09-16T08:05:25.276 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:24 smithi028 ceph-mon[23744]: rocksdb: Options.periodic_compaction_seconds: 0 2024-09-16T08:05:25.276 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:24 smithi028 ceph-mon[23744]: rocksdb: Options.enable_blob_files: false 2024-09-16T08:05:25.276 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:24 smithi028 ceph-mon[23744]: rocksdb: Options.min_blob_size: 0 2024-09-16T08:05:25.277 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:24 smithi028 ceph-mon[23744]: rocksdb: Options.blob_file_size: 268435456 2024-09-16T08:05:25.277 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:24 smithi028 ceph-mon[23744]: rocksdb: Options.blob_compression_type: NoCompression 2024-09-16T08:05:25.277 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:24 smithi028 ceph-mon[23744]: rocksdb: Options.enable_blob_garbage_collection: false 2024-09-16T08:05:25.277 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:24 smithi028 ceph-mon[23744]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-09-16T08:05:25.277 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:24 smithi028 ceph-mon[23744]: rocksdb: [db/version_set.cc:4773] Recovered from manifest file:/var/lib/ceph/mon/ceph-smithi028/store.db/MANIFEST-000009 succeeded,manifest_file_number is 9, next_file_number is 11, last_sequence is 5, log_number is 5,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-09-16T08:05:25.277 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:24 smithi028 ceph-mon[23744]: rocksdb: [db/version_set.cc:4782] Column family [default] (ID 0), log number is 5 2024-09-16T08:05:25.277 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:24 smithi028 ceph-mon[23744]: rocksdb: [db/version_set.cc:4083] Creating manifest 13 2024-09-16T08:05:25.277 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:24 smithi028 ceph-mon[23744]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726473924996155, "job": 1, "event": "recovery_started", "wal_files": [10]} 2024-09-16T08:05:25.277 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:24 smithi028 ceph-mon[23744]: rocksdb: [db/db_impl/db_impl_open.cc:847] Recovering log #10 mode 2 2024-09-16T08:05:25.277 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:24 smithi028 ceph-mon[23744]: rocksdb: [table/block_based/filter_policy.cc:996] Using legacy Bloom filter with high (20) bits/key. Dramatic filter space and/or accuracy improvement is available with format_version>=5. 2024-09-16T08:05:25.277 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:24 smithi028 ceph-mon[23744]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726473924996926, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 14, "file_size": 79395, "file_checksum": "", "file_checksum_func_name": "Unknown", "table_properties": {"data_size": 77659, "index_size": 224, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 581, "raw_key_size": 9884, "raw_average_key_size": 47, "raw_value_size": 71968, "raw_average_value_size": 344, "num_data_blocks": 10, "num_entries": 209, "num_deletions": 3, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "rocksdb.BuiltinBloomFilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; ", "creation_time": 1726473924, "oldest_key_time": 0, "file_creation_time": 0, "db_id": "ec038f90-c9c7-40e5-a834-3ab51b72ee66", "db_session_id": "AJVSL2TIULFTW0OEMD94"}} 2024-09-16T08:05:25.277 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:24 smithi028 ceph-mon[23744]: rocksdb: [db/version_set.cc:4083] Creating manifest 15 2024-09-16T08:05:25.277 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:24 smithi028 ceph-mon[23744]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726473924997290, "job": 1, "event": "recovery_finished"} 2024-09-16T08:05:25.278 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:24 smithi028 ceph-mon[23744]: rocksdb: [file/delete_scheduler.cc:73] Deleted file /var/lib/ceph/mon/ceph-smithi028/store.db/000010.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-09-16T08:05:25.278 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:24 smithi028 ceph-mon[23744]: rocksdb: [db/db_impl/db_impl_open.cc:1701] SstFileManager instance 0x55a279ff6a80 2024-09-16T08:05:25.278 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:24 smithi028 ceph-mon[23744]: rocksdb: DB pointer 0x55a27a006000 2024-09-16T08:05:25.278 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:24 smithi028 ceph-mon[23744]: rocksdb: [db/db_impl/db_impl.cc:902] ------- DUMPING STATS ------- 2024-09-16T08:05:25.278 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:24 smithi028 ceph-mon[23744]: rocksdb: [db/db_impl/db_impl.cc:903] 2024-09-16T08:05:25.278 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout: ** DB Stats ** 2024-09-16T08:05:25.278 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-16T08:05:25.278 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-09-16T08:05:25.278 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-16T08:05:25.278 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-16T08:05:25.278 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-09-16T08:05:25.278 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2024-09-16T08:05:25.278 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-16T08:05:25.278 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout: 2024-09-16T08:05:25.278 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout: ** Compaction Stats [default] ** 2024-09-16T08:05:25.279 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-16T08:05:25.279 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-16T08:05:25.279 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout: L0 2/0 79.33 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 152.7 0.00 0.00 1 0.000 0 0 2024-09-16T08:05:25.279 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout: Sum 2/0 79.33 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 152.7 0.00 0.00 1 0.000 0 0 2024-09-16T08:05:25.279 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 152.7 0.00 0.00 1 0.000 0 0 2024-09-16T08:05:25.279 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout: 2024-09-16T08:05:25.279 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout: ** Compaction Stats [default] ** 2024-09-16T08:05:25.279 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-16T08:05:25.279 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-16T08:05:25.279 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 152.7 0.00 0.00 1 0.000 0 0 2024-09-16T08:05:25.279 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-16T08:05:25.279 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-09-16T08:05:25.279 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-09-16T08:05:25.279 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-09-16T08:05:25.279 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-09-16T08:05:25.280 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout: AddFile(Keys): cumulative 0, interval 0 2024-09-16T08:05:25.280 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout: Cumulative compaction: 0.00 GB write, 24.21 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-16T08:05:25.280 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout: Interval compaction: 0.00 GB write, 24.21 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-16T08:05:25.280 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-09-16T08:05:25.280 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout: 2024-09-16T08:05:25.280 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout: ** File Read Latency Histogram By Level [default] ** 2024-09-16T08:05:25.280 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout: 2024-09-16T08:05:25.280 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout: ** Compaction Stats [default] ** 2024-09-16T08:05:25.280 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-16T08:05:25.280 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-16T08:05:25.280 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout: L0 2/0 79.33 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 152.7 0.00 0.00 1 0.000 0 0 2024-09-16T08:05:25.280 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout: Sum 2/0 79.33 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 152.7 0.00 0.00 1 0.000 0 0 2024-09-16T08:05:25.280 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2024-09-16T08:05:25.280 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout: 2024-09-16T08:05:25.281 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout: ** Compaction Stats [default] ** 2024-09-16T08:05:25.281 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-16T08:05:25.281 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-16T08:05:25.281 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 152.7 0.00 0.00 1 0.000 0 0 2024-09-16T08:05:25.281 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-16T08:05:25.281 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-09-16T08:05:25.281 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-09-16T08:05:25.281 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-09-16T08:05:25.281 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-09-16T08:05:25.281 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout: AddFile(Keys): cumulative 0, interval 0 2024-09-16T08:05:25.281 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout: Cumulative compaction: 0.00 GB write, 23.97 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-16T08:05:25.281 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-16T08:05:25.281 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-09-16T08:05:25.282 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout: 2024-09-16T08:05:25.282 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout: ** File Read Latency Histogram By Level [default] ** 2024-09-16T08:05:25.282 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:24 smithi028 ceph-mon[23744]: starting mon.smithi028 rank 0 at public addrs [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] at bind addrs [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] mon_data /var/lib/ceph/mon/ceph-smithi028 fsid 5622b2fe-7402-11ef-bceb-c7b262605968 2024-09-16T08:05:25.282 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:24 smithi028 ceph-mon[23744]: mon.smithi028@-1(???) e1 preinit fsid 5622b2fe-7402-11ef-bceb-c7b262605968 2024-09-16T08:05:25.282 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:24 smithi028 ceph-mon[23744]: mon.smithi028@-1(???).mds e1 new map 2024-09-16T08:05:25.282 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:24 smithi028 ceph-mon[23744]: mon.smithi028@-1(???).mds e1 print_map 2024-09-16T08:05:25.282 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout: e1 2024-09-16T08:05:25.282 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-09-16T08:05:25.282 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-09-16T08:05:25.282 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout: legacy client fscid: -1 2024-09-16T08:05:25.282 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout: 2024-09-16T08:05:25.282 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout: No filesystems configured 2024-09-16T08:05:25.282 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:25 smithi028 ceph-mon[23744]: mon.smithi028@-1(???).osd e1 crush map has features 3314932999778484224, adjusting msgr requires 2024-09-16T08:05:25.282 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:25 smithi028 ceph-mon[23744]: mon.smithi028@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-09-16T08:05:25.283 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:25 smithi028 ceph-mon[23744]: mon.smithi028@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-09-16T08:05:25.283 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:25 smithi028 ceph-mon[23744]: mon.smithi028@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-09-16T08:05:25.283 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:25 smithi028 ceph-mon[23744]: mon.smithi028@-1(???).paxosservice(auth 1..2) refresh upgraded, format 0 -> 3 2024-09-16T08:05:25.283 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:25 smithi028 bash[23726]: a2f688697052cb0def56c126ac145a22be239a549f59f8d80465942185c8d025 2024-09-16T08:05:25.283 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:25 smithi028 systemd[1]: Started Ceph mon.smithi028 for 5622b2fe-7402-11ef-bceb-c7b262605968. 2024-09-16T08:05:26.104 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:26 smithi028 ceph-mon[23744]: mon.smithi028 is new leader, mons smithi028 in quorum (ranks 0) 2024-09-16T08:05:26.104 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:26 smithi028 ceph-mon[23744]: monmap e1: 1 mons at {smithi028=[v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0]} 2024-09-16T08:05:26.104 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:26 smithi028 ceph-mon[23744]: fsmap 2024-09-16T08:05:26.104 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:26 smithi028 ceph-mon[23744]: osdmap e1: 0 total, 0 up, 0 in 2024-09-16T08:05:26.104 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:26 smithi028 ceph-mon[23744]: mgrmap e1: no daemons active 2024-09-16T08:05:26.104 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:26 smithi028 ceph-mon[23744]: from='client.? 172.21.15.28:0/4154209810' entity='client.admin' 2024-09-16T08:05:26.106 INFO:teuthology.orchestra.run.smithi028.stderr:Wrote config to /etc/ceph/ceph.conf 2024-09-16T08:05:26.106 INFO:teuthology.orchestra.run.smithi028.stderr:Wrote keyring to /etc/ceph/ceph.client.admin.keyring 2024-09-16T08:05:26.107 INFO:teuthology.orchestra.run.smithi028.stderr:Creating mgr... 2024-09-16T08:05:26.107 INFO:teuthology.orchestra.run.smithi028.stderr:Verifying port 9283 ... 2024-09-16T08:05:26.296 INFO:teuthology.orchestra.run.smithi028.stderr:systemctl: Failed to reset failed state of unit ceph-5622b2fe-7402-11ef-bceb-c7b262605968@mgr.smithi028.azvyzy.service: Unit ceph-5622b2fe-7402-11ef-bceb-c7b262605968@mgr.smithi028.azvyzy.service not loaded. 2024-09-16T08:05:26.304 INFO:teuthology.orchestra.run.smithi028.stderr:systemctl: Created symlink /etc/systemd/system/ceph-5622b2fe-7402-11ef-bceb-c7b262605968.target.wants/ceph-5622b2fe-7402-11ef-bceb-c7b262605968@mgr.smithi028.azvyzy.service → /etc/systemd/system/ceph-5622b2fe-7402-11ef-bceb-c7b262605968@.service. 2024-09-16T08:05:27.004 INFO:teuthology.orchestra.run.smithi028.stderr:systemctl: disabled 2024-09-16T08:05:27.012 INFO:teuthology.orchestra.run.smithi028.stderr:systemctl: inactive 2024-09-16T08:05:27.013 INFO:teuthology.orchestra.run.smithi028.stderr:firewalld.service is not enabled 2024-09-16T08:05:27.013 INFO:teuthology.orchestra.run.smithi028.stderr:Not possible to enable service . firewalld.service is not available 2024-09-16T08:05:27.023 INFO:teuthology.orchestra.run.smithi028.stderr:systemctl: disabled 2024-09-16T08:05:27.031 INFO:teuthology.orchestra.run.smithi028.stderr:systemctl: inactive 2024-09-16T08:05:27.032 INFO:teuthology.orchestra.run.smithi028.stderr:firewalld.service is not enabled 2024-09-16T08:05:27.032 INFO:teuthology.orchestra.run.smithi028.stderr:Not possible to open ports <[9283]>. firewalld.service is not available 2024-09-16T08:05:27.032 INFO:teuthology.orchestra.run.smithi028.stderr:Waiting for mgr to start... 2024-09-16T08:05:27.032 INFO:teuthology.orchestra.run.smithi028.stderr:Waiting for mgr... 2024-09-16T08:05:27.558 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: 2024-09-16T08:05:27.558 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: { 2024-09-16T08:05:27.558 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: "fsid": "5622b2fe-7402-11ef-bceb-c7b262605968", 2024-09-16T08:05:27.558 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: "health": { 2024-09-16T08:05:27.558 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: "status": "HEALTH_OK", 2024-09-16T08:05:27.558 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: "checks": {}, 2024-09-16T08:05:27.558 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: "mutes": [] 2024-09-16T08:05:27.558 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: }, 2024-09-16T08:05:27.558 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: "election_epoch": 5, 2024-09-16T08:05:27.559 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: "quorum": [ 2024-09-16T08:05:27.559 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: 0 2024-09-16T08:05:27.559 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: ], 2024-09-16T08:05:27.559 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: "quorum_names": [ 2024-09-16T08:05:27.559 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: "smithi028" 2024-09-16T08:05:27.559 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: ], 2024-09-16T08:05:27.559 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: "quorum_age": 2, 2024-09-16T08:05:27.559 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: "monmap": { 2024-09-16T08:05:27.559 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-16T08:05:27.559 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: "min_mon_release_name": "quincy", 2024-09-16T08:05:27.559 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: "num_mons": 1 2024-09-16T08:05:27.559 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: }, 2024-09-16T08:05:27.559 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: "osdmap": { 2024-09-16T08:05:27.559 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-16T08:05:27.560 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: "num_osds": 0, 2024-09-16T08:05:27.560 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: "num_up_osds": 0, 2024-09-16T08:05:27.560 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: "osd_up_since": 0, 2024-09-16T08:05:27.560 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: "num_in_osds": 0, 2024-09-16T08:05:27.560 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: "osd_in_since": 0, 2024-09-16T08:05:27.560 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: "num_remapped_pgs": 0 2024-09-16T08:05:27.560 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: }, 2024-09-16T08:05:27.560 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: "pgmap": { 2024-09-16T08:05:27.560 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: "pgs_by_state": [], 2024-09-16T08:05:27.560 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: "num_pgs": 0, 2024-09-16T08:05:27.560 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: "num_pools": 0, 2024-09-16T08:05:27.560 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: "num_objects": 0, 2024-09-16T08:05:27.560 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: "data_bytes": 0, 2024-09-16T08:05:27.561 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: "bytes_used": 0, 2024-09-16T08:05:27.561 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: "bytes_avail": 0, 2024-09-16T08:05:27.561 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: "bytes_total": 0 2024-09-16T08:05:27.561 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: }, 2024-09-16T08:05:27.561 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: "fsmap": { 2024-09-16T08:05:27.561 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-16T08:05:27.561 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: "by_rank": [], 2024-09-16T08:05:27.561 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: "up:standby": 0 2024-09-16T08:05:27.561 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: }, 2024-09-16T08:05:27.561 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: "mgrmap": { 2024-09-16T08:05:27.561 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: "available": false, 2024-09-16T08:05:27.561 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: "num_standbys": 0, 2024-09-16T08:05:27.561 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: "modules": [ 2024-09-16T08:05:27.561 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: "iostat", 2024-09-16T08:05:27.561 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: "nfs", 2024-09-16T08:05:27.562 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: "restful" 2024-09-16T08:05:27.562 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: ], 2024-09-16T08:05:27.562 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: "services": {} 2024-09-16T08:05:27.562 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: }, 2024-09-16T08:05:27.562 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: "servicemap": { 2024-09-16T08:05:27.562 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-16T08:05:27.562 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: "modified": "2024-09-16T08:05:20.224659+0000", 2024-09-16T08:05:27.562 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: "services": {} 2024-09-16T08:05:27.562 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: }, 2024-09-16T08:05:27.562 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: "progress_events": {} 2024-09-16T08:05:27.562 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: } 2024-09-16T08:05:27.697 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:27 smithi028 ceph-mon[23744]: from='client.? 172.21.15.28:0/3909052346' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-09-16T08:05:28.140 INFO:teuthology.orchestra.run.smithi028.stderr:mgr not available, waiting (1/15)... 2024-09-16T08:05:30.647 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: 2024-09-16T08:05:30.647 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: { 2024-09-16T08:05:30.648 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: "fsid": "5622b2fe-7402-11ef-bceb-c7b262605968", 2024-09-16T08:05:30.648 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: "health": { 2024-09-16T08:05:30.648 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: "status": "HEALTH_OK", 2024-09-16T08:05:30.648 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: "checks": {}, 2024-09-16T08:05:30.648 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: "mutes": [] 2024-09-16T08:05:30.648 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: }, 2024-09-16T08:05:30.648 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: "election_epoch": 5, 2024-09-16T08:05:30.648 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: "quorum": [ 2024-09-16T08:05:30.648 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: 0 2024-09-16T08:05:30.648 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: ], 2024-09-16T08:05:30.648 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: "quorum_names": [ 2024-09-16T08:05:30.648 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: "smithi028" 2024-09-16T08:05:30.648 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: ], 2024-09-16T08:05:30.649 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: "quorum_age": 5, 2024-09-16T08:05:30.649 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: "monmap": { 2024-09-16T08:05:30.649 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-16T08:05:30.649 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: "min_mon_release_name": "quincy", 2024-09-16T08:05:30.649 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: "num_mons": 1 2024-09-16T08:05:30.649 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: }, 2024-09-16T08:05:30.649 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: "osdmap": { 2024-09-16T08:05:30.649 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-16T08:05:30.649 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: "num_osds": 0, 2024-09-16T08:05:30.649 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: "num_up_osds": 0, 2024-09-16T08:05:30.649 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: "osd_up_since": 0, 2024-09-16T08:05:30.649 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: "num_in_osds": 0, 2024-09-16T08:05:30.649 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: "osd_in_since": 0, 2024-09-16T08:05:30.649 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: "num_remapped_pgs": 0 2024-09-16T08:05:30.650 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: }, 2024-09-16T08:05:30.650 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: "pgmap": { 2024-09-16T08:05:30.650 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: "pgs_by_state": [], 2024-09-16T08:05:30.650 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: "num_pgs": 0, 2024-09-16T08:05:30.650 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: "num_pools": 0, 2024-09-16T08:05:30.650 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: "num_objects": 0, 2024-09-16T08:05:30.650 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: "data_bytes": 0, 2024-09-16T08:05:30.650 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: "bytes_used": 0, 2024-09-16T08:05:30.650 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: "bytes_avail": 0, 2024-09-16T08:05:30.650 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: "bytes_total": 0 2024-09-16T08:05:30.650 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: }, 2024-09-16T08:05:30.650 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: "fsmap": { 2024-09-16T08:05:30.650 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-16T08:05:30.651 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: "by_rank": [], 2024-09-16T08:05:30.651 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: "up:standby": 0 2024-09-16T08:05:30.651 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: }, 2024-09-16T08:05:30.651 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: "mgrmap": { 2024-09-16T08:05:30.651 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: "available": false, 2024-09-16T08:05:30.651 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: "num_standbys": 0, 2024-09-16T08:05:30.651 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: "modules": [ 2024-09-16T08:05:30.651 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: "iostat", 2024-09-16T08:05:30.651 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: "nfs", 2024-09-16T08:05:30.651 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: "restful" 2024-09-16T08:05:30.651 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: ], 2024-09-16T08:05:30.651 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: "services": {} 2024-09-16T08:05:30.651 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: }, 2024-09-16T08:05:30.651 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: "servicemap": { 2024-09-16T08:05:30.651 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-16T08:05:30.652 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: "modified": "2024-09-16T08:05:20.224659+0000", 2024-09-16T08:05:30.652 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: "services": {} 2024-09-16T08:05:30.652 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: }, 2024-09-16T08:05:30.652 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: "progress_events": {} 2024-09-16T08:05:30.652 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: } 2024-09-16T08:05:31.110 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:31 smithi028 ceph-mon[23744]: from='client.? 172.21.15.28:0/2862497377' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-09-16T08:05:31.113 INFO:teuthology.orchestra.run.smithi028.stderr:mgr not available, waiting (2/15)... 2024-09-16T08:05:33.538 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:33 smithi028 ceph-mon[23744]: Activating manager daemon smithi028.azvyzy 2024-09-16T08:05:33.538 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:33 smithi028 ceph-mon[23744]: mgrmap e2: smithi028.azvyzy(active, starting, since 0.00276397s) 2024-09-16T08:05:33.538 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:33 smithi028 ceph-mon[23744]: from='mgr.14100 172.21.15.28:0/3156209474' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-16T08:05:33.538 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:33 smithi028 ceph-mon[23744]: from='mgr.14100 172.21.15.28:0/3156209474' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-16T08:05:33.538 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:33 smithi028 ceph-mon[23744]: from='mgr.14100 172.21.15.28:0/3156209474' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-16T08:05:33.538 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:33 smithi028 ceph-mon[23744]: from='mgr.14100 172.21.15.28:0/3156209474' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "mon metadata", "id": "smithi028"}]: dispatch 2024-09-16T08:05:33.538 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:33 smithi028 ceph-mon[23744]: from='mgr.14100 172.21.15.28:0/3156209474' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "mgr metadata", "who": "smithi028.azvyzy", "id": "smithi028.azvyzy"}]: dispatch 2024-09-16T08:05:33.539 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:33 smithi028 ceph-mon[23744]: Manager daemon smithi028.azvyzy is now available 2024-09-16T08:05:33.539 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:33 smithi028 ceph-mon[23744]: from='mgr.14100 172.21.15.28:0/3156209474' entity='mgr.smithi028.azvyzy' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi028.azvyzy/mirror_snapshot_schedule"}]: dispatch 2024-09-16T08:05:33.539 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:33 smithi028 ceph-mon[23744]: from='mgr.14100 172.21.15.28:0/3156209474' entity='mgr.smithi028.azvyzy' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi028.azvyzy/trash_purge_schedule"}]: dispatch 2024-09-16T08:05:33.539 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:33 smithi028 ceph-mon[23744]: from='mgr.14100 172.21.15.28:0/3156209474' entity='mgr.smithi028.azvyzy' 2024-09-16T08:05:33.539 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:33 smithi028 ceph-mon[23744]: from='mgr.14100 172.21.15.28:0/3156209474' entity='mgr.smithi028.azvyzy' 2024-09-16T08:05:33.539 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:33 smithi028 ceph-mon[23744]: from='mgr.14100 172.21.15.28:0/3156209474' entity='mgr.smithi028.azvyzy' 2024-09-16T08:05:33.660 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: 2024-09-16T08:05:33.660 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: { 2024-09-16T08:05:33.660 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: "fsid": "5622b2fe-7402-11ef-bceb-c7b262605968", 2024-09-16T08:05:33.660 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: "health": { 2024-09-16T08:05:33.660 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: "status": "HEALTH_OK", 2024-09-16T08:05:33.660 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: "checks": {}, 2024-09-16T08:05:33.660 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: "mutes": [] 2024-09-16T08:05:33.660 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: }, 2024-09-16T08:05:33.660 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: "election_epoch": 5, 2024-09-16T08:05:33.660 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: "quorum": [ 2024-09-16T08:05:33.661 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: 0 2024-09-16T08:05:33.661 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: ], 2024-09-16T08:05:33.661 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: "quorum_names": [ 2024-09-16T08:05:33.661 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: "smithi028" 2024-09-16T08:05:33.661 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: ], 2024-09-16T08:05:33.661 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: "quorum_age": 8, 2024-09-16T08:05:33.661 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: "monmap": { 2024-09-16T08:05:33.661 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-16T08:05:33.661 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: "min_mon_release_name": "quincy", 2024-09-16T08:05:33.661 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: "num_mons": 1 2024-09-16T08:05:33.661 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: }, 2024-09-16T08:05:33.661 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: "osdmap": { 2024-09-16T08:05:33.661 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-16T08:05:33.661 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: "num_osds": 0, 2024-09-16T08:05:33.662 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: "num_up_osds": 0, 2024-09-16T08:05:33.662 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: "osd_up_since": 0, 2024-09-16T08:05:33.666 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: "num_in_osds": 0, 2024-09-16T08:05:33.666 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: "osd_in_since": 0, 2024-09-16T08:05:33.666 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: "num_remapped_pgs": 0 2024-09-16T08:05:33.666 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: }, 2024-09-16T08:05:33.666 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: "pgmap": { 2024-09-16T08:05:33.667 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: "pgs_by_state": [], 2024-09-16T08:05:33.667 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: "num_pgs": 0, 2024-09-16T08:05:33.667 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: "num_pools": 0, 2024-09-16T08:05:33.667 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: "num_objects": 0, 2024-09-16T08:05:33.667 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: "data_bytes": 0, 2024-09-16T08:05:33.667 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: "bytes_used": 0, 2024-09-16T08:05:33.667 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: "bytes_avail": 0, 2024-09-16T08:05:33.668 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: "bytes_total": 0 2024-09-16T08:05:33.668 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: }, 2024-09-16T08:05:33.668 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: "fsmap": { 2024-09-16T08:05:33.668 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-16T08:05:33.668 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: "by_rank": [], 2024-09-16T08:05:33.668 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: "up:standby": 0 2024-09-16T08:05:33.668 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: }, 2024-09-16T08:05:33.668 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: "mgrmap": { 2024-09-16T08:05:33.669 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: "available": false, 2024-09-16T08:05:33.669 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: "num_standbys": 0, 2024-09-16T08:05:33.669 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: "modules": [ 2024-09-16T08:05:33.669 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: "iostat", 2024-09-16T08:05:33.669 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: "nfs", 2024-09-16T08:05:33.669 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: "restful" 2024-09-16T08:05:33.669 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: ], 2024-09-16T08:05:33.670 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: "services": {} 2024-09-16T08:05:33.670 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: }, 2024-09-16T08:05:33.670 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: "servicemap": { 2024-09-16T08:05:33.670 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-16T08:05:33.670 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: "modified": "2024-09-16T08:05:20.224659+0000", 2024-09-16T08:05:33.670 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: "services": {} 2024-09-16T08:05:33.670 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: }, 2024-09-16T08:05:33.670 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: "progress_events": {} 2024-09-16T08:05:33.670 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: } 2024-09-16T08:05:34.218 INFO:teuthology.orchestra.run.smithi028.stderr:mgr not available, waiting (3/15)... 2024-09-16T08:05:34.696 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:34 smithi028 ceph-mon[23744]: from='client.? 172.21.15.28:0/2898822219' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-09-16T08:05:34.696 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:34 smithi028 ceph-mon[23744]: mgrmap e3: smithi028.azvyzy(active, since 1.00612s) 2024-09-16T08:05:36.869 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: 2024-09-16T08:05:36.869 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: { 2024-09-16T08:05:36.869 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: "fsid": "5622b2fe-7402-11ef-bceb-c7b262605968", 2024-09-16T08:05:36.869 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: "health": { 2024-09-16T08:05:36.870 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: "status": "HEALTH_OK", 2024-09-16T08:05:36.870 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: "checks": {}, 2024-09-16T08:05:36.870 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: "mutes": [] 2024-09-16T08:05:36.870 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: }, 2024-09-16T08:05:36.870 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: "election_epoch": 5, 2024-09-16T08:05:36.870 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: "quorum": [ 2024-09-16T08:05:36.870 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: 0 2024-09-16T08:05:36.870 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: ], 2024-09-16T08:05:36.870 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: "quorum_names": [ 2024-09-16T08:05:36.870 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: "smithi028" 2024-09-16T08:05:36.870 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: ], 2024-09-16T08:05:36.870 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: "quorum_age": 11, 2024-09-16T08:05:36.871 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: "monmap": { 2024-09-16T08:05:36.871 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-16T08:05:36.871 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: "min_mon_release_name": "quincy", 2024-09-16T08:05:36.871 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: "num_mons": 1 2024-09-16T08:05:36.871 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: }, 2024-09-16T08:05:36.871 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: "osdmap": { 2024-09-16T08:05:36.871 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-16T08:05:36.871 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: "num_osds": 0, 2024-09-16T08:05:36.871 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: "num_up_osds": 0, 2024-09-16T08:05:36.871 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: "osd_up_since": 0, 2024-09-16T08:05:36.871 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: "num_in_osds": 0, 2024-09-16T08:05:36.875 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: "osd_in_since": 0, 2024-09-16T08:05:36.875 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: "num_remapped_pgs": 0 2024-09-16T08:05:36.875 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: }, 2024-09-16T08:05:36.875 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: "pgmap": { 2024-09-16T08:05:36.875 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: "pgs_by_state": [], 2024-09-16T08:05:36.875 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: "num_pgs": 0, 2024-09-16T08:05:36.875 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: "num_pools": 0, 2024-09-16T08:05:36.875 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: "num_objects": 0, 2024-09-16T08:05:36.875 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: "data_bytes": 0, 2024-09-16T08:05:36.875 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: "bytes_used": 0, 2024-09-16T08:05:36.875 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: "bytes_avail": 0, 2024-09-16T08:05:36.876 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: "bytes_total": 0 2024-09-16T08:05:36.876 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: }, 2024-09-16T08:05:36.876 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: "fsmap": { 2024-09-16T08:05:36.876 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-16T08:05:36.876 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: "by_rank": [], 2024-09-16T08:05:36.876 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: "up:standby": 0 2024-09-16T08:05:36.876 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: }, 2024-09-16T08:05:36.876 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: "mgrmap": { 2024-09-16T08:05:36.876 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: "available": true, 2024-09-16T08:05:36.876 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: "num_standbys": 0, 2024-09-16T08:05:36.876 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: "modules": [ 2024-09-16T08:05:36.876 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: "iostat", 2024-09-16T08:05:36.876 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: "nfs", 2024-09-16T08:05:36.877 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: "restful" 2024-09-16T08:05:36.877 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: ], 2024-09-16T08:05:36.877 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: "services": {} 2024-09-16T08:05:36.877 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: }, 2024-09-16T08:05:36.877 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: "servicemap": { 2024-09-16T08:05:36.877 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-16T08:05:36.877 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: "modified": "2024-09-16T08:05:20.224659+0000", 2024-09-16T08:05:36.877 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: "services": {} 2024-09-16T08:05:36.877 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: }, 2024-09-16T08:05:36.877 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: "progress_events": {} 2024-09-16T08:05:36.877 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: } 2024-09-16T08:05:37.196 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:37 smithi028 ceph-mon[23744]: mgrmap e4: smithi028.azvyzy(active, since 2s) 2024-09-16T08:05:37.196 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:37 smithi028 ceph-mon[23744]: from='client.? 172.21.15.28:0/1025454084' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-09-16T08:05:37.382 INFO:teuthology.orchestra.run.smithi028.stderr:mgr is available 2024-09-16T08:05:37.951 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: 2024-09-16T08:05:37.951 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: [global] 2024-09-16T08:05:37.951 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: fsid = 5622b2fe-7402-11ef-bceb-c7b262605968 2024-09-16T08:05:37.951 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: mon_osd_allow_pg_remap = true 2024-09-16T08:05:37.951 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: mon_osd_allow_primary_affinity = true 2024-09-16T08:05:37.951 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: mon_warn_on_no_sortbitwise = false 2024-09-16T08:05:37.951 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: osd_crush_chooseleaf_type = 0 2024-09-16T08:05:37.951 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: 2024-09-16T08:05:37.951 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: [mgr] 2024-09-16T08:05:37.951 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: mgr/telemetry/nag = false 2024-09-16T08:05:37.951 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: 2024-09-16T08:05:37.952 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: [osd] 2024-09-16T08:05:37.952 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: osd_map_max_advance = 10 2024-09-16T08:05:37.952 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: osd_sloppy_crc = true 2024-09-16T08:05:38.345 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:38 smithi028 ceph-mon[23744]: from='client.? 172.21.15.28:0/2030723979' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2024-09-16T08:05:38.421 INFO:teuthology.orchestra.run.smithi028.stderr:Enabling cephadm module... 2024-09-16T08:05:39.416 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:39 smithi028 ceph-mon[23744]: from='client.? 172.21.15.28:0/2649035187' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "cephadm"}]: dispatch 2024-09-16T08:05:40.321 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:40 smithi028 ceph-mon[23744]: from='client.? 172.21.15.28:0/2649035187' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "cephadm"}]': finished 2024-09-16T08:05:40.321 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:40 smithi028 ceph-mon[23744]: mgrmap e5: smithi028.azvyzy(active, since 5s) 2024-09-16T08:05:40.326 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: { 2024-09-16T08:05:40.326 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: "epoch": 5, 2024-09-16T08:05:40.326 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: "available": true, 2024-09-16T08:05:40.326 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: "active_name": "smithi028.azvyzy", 2024-09-16T08:05:40.326 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: "num_standby": 0 2024-09-16T08:05:40.326 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: } 2024-09-16T08:05:40.824 INFO:teuthology.orchestra.run.smithi028.stderr:Waiting for the mgr to restart... 2024-09-16T08:05:40.824 INFO:teuthology.orchestra.run.smithi028.stderr:Waiting for mgr epoch 5... 2024-09-16T08:05:41.196 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:41 smithi028 ceph-mon[23744]: from='client.? 172.21.15.28:0/1745309528' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2024-09-16T08:05:46.946 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:46 smithi028 ceph-mon[23744]: Active manager daemon smithi028.azvyzy restarted 2024-09-16T08:05:46.947 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:46 smithi028 ceph-mon[23744]: Activating manager daemon smithi028.azvyzy 2024-09-16T08:05:46.947 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:46 smithi028 ceph-mon[23744]: osdmap e2: 0 total, 0 up, 0 in 2024-09-16T08:05:46.947 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:46 smithi028 ceph-mon[23744]: mgrmap e6: smithi028.azvyzy(active, starting, since 0.545241s) 2024-09-16T08:05:46.947 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:46 smithi028 ceph-mon[23744]: from='mgr.14120 172.21.15.28:0/3751098848' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "mon metadata", "id": "smithi028"}]: dispatch 2024-09-16T08:05:46.947 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:46 smithi028 ceph-mon[23744]: from='mgr.14120 172.21.15.28:0/3751098848' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "mgr metadata", "who": "smithi028.azvyzy", "id": "smithi028.azvyzy"}]: dispatch 2024-09-16T08:05:46.947 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:46 smithi028 ceph-mon[23744]: from='mgr.14120 172.21.15.28:0/3751098848' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-16T08:05:46.947 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:46 smithi028 ceph-mon[23744]: from='mgr.14120 172.21.15.28:0/3751098848' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-16T08:05:46.947 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:46 smithi028 ceph-mon[23744]: from='mgr.14120 172.21.15.28:0/3751098848' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-16T08:05:46.947 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:46 smithi028 ceph-mon[23744]: Manager daemon smithi028.azvyzy is now available 2024-09-16T08:05:46.947 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:46 smithi028 ceph-mon[23744]: from='mgr.14120 172.21.15.28:0/3751098848' entity='mgr.smithi028.azvyzy' 2024-09-16T08:05:46.948 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:46 smithi028 ceph-mon[23744]: from='mgr.14120 172.21.15.28:0/3751098848' entity='mgr.smithi028.azvyzy' 2024-09-16T08:05:46.948 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:46 smithi028 ceph-mon[23744]: from='mgr.14120 172.21.15.28:0/3751098848' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-16T08:05:46.948 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:46 smithi028 ceph-mon[23744]: from='mgr.14120 172.21.15.28:0/3751098848' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-16T08:05:46.948 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:46 smithi028 ceph-mon[23744]: from='mgr.14120 172.21.15.28:0/3751098848' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-16T08:05:46.948 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:46 smithi028 ceph-mon[23744]: from='mgr.14120 172.21.15.28:0/3751098848' entity='mgr.smithi028.azvyzy' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi028.azvyzy/mirror_snapshot_schedule"}]: dispatch 2024-09-16T08:05:46.948 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:46 smithi028 ceph-mon[23744]: from='mgr.14120 172.21.15.28:0/3751098848' entity='mgr.smithi028.azvyzy' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi028.azvyzy/trash_purge_schedule"}]: dispatch 2024-09-16T08:05:47.015 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: { 2024-09-16T08:05:47.015 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: "mgrmap_epoch": 7, 2024-09-16T08:05:47.015 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: "initialized": true 2024-09-16T08:05:47.015 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: } 2024-09-16T08:05:47.638 INFO:teuthology.orchestra.run.smithi028.stderr:mgr epoch 5 is available 2024-09-16T08:05:47.638 INFO:teuthology.orchestra.run.smithi028.stderr:Setting orchestrator backend to cephadm... 2024-09-16T08:05:48.196 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:48 smithi028 ceph-mon[23744]: from='client.14124 -' entity='client.admin' cmd=[{"prefix": "get_command_descriptions"}]: dispatch 2024-09-16T08:05:48.196 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:48 smithi028 ceph-mon[23744]: mgrmap e7: smithi028.azvyzy(active, since 1.54786s) 2024-09-16T08:05:48.196 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:48 smithi028 ceph-mon[23744]: from='client.14124 -' entity='client.admin' cmd=[{"prefix": "mgr_status"}]: dispatch 2024-09-16T08:05:49.319 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:49 smithi028 ceph-mon[23744]: [16/Sep/2024:08:05:47] ENGINE Bus STARTING 2024-09-16T08:05:49.320 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:49 smithi028 ceph-mon[23744]: mgrmap e8: smithi028.azvyzy(active, since 2s) 2024-09-16T08:05:49.320 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:49 smithi028 ceph-mon[23744]: [16/Sep/2024:08:05:48] ENGINE Serving on https://172.21.15.28:7150 2024-09-16T08:05:49.320 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:49 smithi028 ceph-mon[23744]: [16/Sep/2024:08:05:48] ENGINE Bus STARTED 2024-09-16T08:05:49.320 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:49 smithi028 ceph-mon[23744]: from='mgr.14120 172.21.15.28:0/3751098848' entity='mgr.smithi028.azvyzy' 2024-09-16T08:05:49.320 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:49 smithi028 ceph-mon[23744]: from='mgr.14120 172.21.15.28:0/3751098848' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-16T08:05:49.320 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:49 smithi028 ceph-mon[23744]: from='mgr.14120 172.21.15.28:0/3751098848' entity='mgr.smithi028.azvyzy' 2024-09-16T08:05:49.320 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:49 smithi028 ceph-mon[23744]: from='mgr.14120 172.21.15.28:0/3751098848' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-16T08:05:49.321 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: value unchanged 2024-09-16T08:05:49.875 INFO:teuthology.orchestra.run.smithi028.stderr:Generating ssh key... 2024-09-16T08:05:50.146 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:50 smithi028 ceph-mon[23744]: from='client.14132 -' entity='client.admin' cmd=[{"prefix": "orch set backend", "module_name": "cephadm", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T08:05:51.378 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:51 smithi028 ceph-mon[23744]: from='client.14134 -' entity='client.admin' cmd=[{"prefix": "cephadm set-user", "user": "root", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T08:05:51.378 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:51 smithi028 ceph-mon[23744]: from='mgr.14120 172.21.15.28:0/3751098848' entity='mgr.smithi028.azvyzy' 2024-09-16T08:05:51.378 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:51 smithi028 ceph-mon[23744]: from='mgr.14120 172.21.15.28:0/3751098848' entity='mgr.smithi028.azvyzy' 2024-09-16T08:05:52.018 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCjplYqXUwyJsfnlJ6ByM+XAVT+9+Mb8iOLhsJmwBFz3irxfG5OSAMBrK9e6JkgwdH1YQvYmgNELlSTIZRwKYE6gd33FeASu5iEZbaHLDFuGJuBEuHmjOBxSX9bNo/BRPztTXBxzUvI+3CpzmJXqeB1+8fAbqR+BPH24cMNvygng0dvcbBAYT++7Nhwsv/7DjHf9CPrfSvdoNBZJbMa/o6mNsrkRnEW/7SKtlq1SBksXQN/mHRH8ADAqpyLk8qavDa6+DHSXKSGEvNIr9OxTZme946DaUZOai8RZPC1OWsSWBQ5p9rJgCaV4PT8+SN+NXXEIdH09ufDpuPMwfUz8GuWZo/1K9ihDXDsbsNdDEjgptRovpc22k0pGSBxJZnDODtTol2VB9MahAbFJvwhtl+brvwGEjUeRJ+Bf4BipYPN/ExoQnPnvlLpNf05aGO9Tzgl1ExlSzBijiCz+WmeojoMV6JdvuadoXASQmMLUVR4hlJvPfJNakSy6CyBpX1cNAc= ceph-5622b2fe-7402-11ef-bceb-c7b262605968 2024-09-16T08:05:52.446 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:52 smithi028 ceph-mon[23744]: from='client.14136 -' entity='client.admin' cmd=[{"prefix": "cephadm generate-key", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T08:05:52.446 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:52 smithi028 ceph-mon[23744]: Generating ssh key... 2024-09-16T08:05:52.452 INFO:teuthology.orchestra.run.smithi028.stderr:Wrote public SSH key to /home/ubuntu/cephtest/ceph.pub 2024-09-16T08:05:52.452 INFO:teuthology.orchestra.run.smithi028.stderr:Adding key to root@localhost authorized_keys... 2024-09-16T08:05:52.463 INFO:teuthology.orchestra.run.smithi028.stderr:Adding host smithi028... 2024-09-16T08:05:53.264 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:53 smithi028 ceph-mon[23744]: from='client.14138 -' entity='client.admin' cmd=[{"prefix": "cephadm get-pub-key", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T08:05:53.960 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: Added host 'smithi028' with addr '172.21.15.28' 2024-09-16T08:05:54.282 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:54 smithi028 ceph-mon[23744]: from='client.14140 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi028", "addr": "172.21.15.28", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T08:05:54.282 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:54 smithi028 ceph-mon[23744]: from='mgr.14120 172.21.15.28:0/3751098848' entity='mgr.smithi028.azvyzy' 2024-09-16T08:05:54.282 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:54 smithi028 ceph-mon[23744]: from='mgr.14120 172.21.15.28:0/3751098848' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-16T08:05:54.517 INFO:teuthology.orchestra.run.smithi028.stderr:Deploying mon service with default placement... 2024-09-16T08:05:55.115 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: Scheduled mon update... 2024-09-16T08:05:55.446 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:55 smithi028 ceph-mon[23744]: Deploying cephadm binary to smithi028 2024-09-16T08:05:55.446 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:55 smithi028 ceph-mon[23744]: Added host smithi028 2024-09-16T08:05:56.026 INFO:teuthology.orchestra.run.smithi028.stderr:Deploying mgr service with default placement... 2024-09-16T08:05:56.394 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:56 smithi028 ceph-mon[23744]: from='client.14142 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T08:05:56.394 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:56 smithi028 ceph-mon[23744]: Saving service mon spec with placement count:5 2024-09-16T08:05:56.394 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:56 smithi028 ceph-mon[23744]: from='mgr.14120 172.21.15.28:0/3751098848' entity='mgr.smithi028.azvyzy' 2024-09-16T08:05:56.956 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: Scheduled mgr update... 2024-09-16T08:05:57.543 INFO:teuthology.orchestra.run.smithi028.stderr:Deploying crash service with default placement... 2024-09-16T08:05:58.115 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:57 smithi028 ceph-mon[23744]: from='client.14144 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T08:05:58.115 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:57 smithi028 ceph-mon[23744]: Saving service mgr spec with placement count:2 2024-09-16T08:05:58.115 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:57 smithi028 ceph-mon[23744]: from='mgr.14120 172.21.15.28:0/3751098848' entity='mgr.smithi028.azvyzy' 2024-09-16T08:05:58.116 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: Scheduled crash update... 2024-09-16T08:05:58.809 INFO:teuthology.orchestra.run.smithi028.stderr:Deploying prometheus service with default placement... 2024-09-16T08:05:59.363 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:59 smithi028 ceph-mon[23744]: from='client.14146 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "crash", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T08:05:59.363 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:59 smithi028 ceph-mon[23744]: Saving service crash spec with placement * 2024-09-16T08:05:59.364 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:59 smithi028 ceph-mon[23744]: from='mgr.14120 172.21.15.28:0/3751098848' entity='mgr.smithi028.azvyzy' 2024-09-16T08:05:59.364 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:05:59 smithi028 ceph-mon[23744]: from='mgr.14120 172.21.15.28:0/3751098848' entity='mgr.smithi028.azvyzy' 2024-09-16T08:05:59.459 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: Scheduled prometheus update... 2024-09-16T08:06:00.140 INFO:teuthology.orchestra.run.smithi028.stderr:Deploying grafana service with default placement... 2024-09-16T08:06:00.499 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:06:00 smithi028 ceph-mon[23744]: from='mgr.14120 172.21.15.28:0/3751098848' entity='mgr.smithi028.azvyzy' 2024-09-16T08:06:00.499 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:06:00 smithi028 ceph-mon[23744]: from='client.14148 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T08:06:00.499 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:06:00 smithi028 ceph-mon[23744]: Saving service prometheus spec with placement count:1 2024-09-16T08:06:00.499 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:06:00 smithi028 ceph-mon[23744]: from='mgr.14120 172.21.15.28:0/3751098848' entity='mgr.smithi028.azvyzy' 2024-09-16T08:06:00.805 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: Scheduled grafana update... 2024-09-16T08:06:01.891 INFO:teuthology.orchestra.run.smithi028.stderr:Deploying node-exporter service with default placement... 2024-09-16T08:06:02.096 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:06:01 smithi028 ceph-mon[23744]: from='client.14150 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T08:06:02.096 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:06:01 smithi028 ceph-mon[23744]: Saving service grafana spec with placement count:1 2024-09-16T08:06:02.096 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:06:01 smithi028 ceph-mon[23744]: from='mgr.14120 172.21.15.28:0/3751098848' entity='mgr.smithi028.azvyzy' 2024-09-16T08:06:02.096 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:06:01 smithi028 ceph-mon[23744]: from='mgr.14120 172.21.15.28:0/3751098848' entity='mgr.smithi028.azvyzy' 2024-09-16T08:06:02.518 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: Scheduled node-exporter update... 2024-09-16T08:06:03.055 INFO:teuthology.orchestra.run.smithi028.stderr:Deploying alertmanager service with default placement... 2024-09-16T08:06:03.682 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:06:03 smithi028 ceph-mon[23744]: from='client.14152 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "node-exporter", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T08:06:03.682 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:06:03 smithi028 ceph-mon[23744]: Saving service node-exporter spec with placement * 2024-09-16T08:06:03.682 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:06:03 smithi028 ceph-mon[23744]: from='mgr.14120 172.21.15.28:0/3751098848' entity='mgr.smithi028.azvyzy' 2024-09-16T08:06:03.682 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: Scheduled alertmanager update... 2024-09-16T08:06:04.622 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:06:04 smithi028 ceph-mon[23744]: from='mgr.14120 172.21.15.28:0/3751098848' entity='mgr.smithi028.azvyzy' 2024-09-16T08:06:04.622 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:06:04 smithi028 ceph-mon[23744]: from='client.14154 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T08:06:04.622 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:06:04 smithi028 ceph-mon[23744]: Saving service alertmanager spec with placement count:1 2024-09-16T08:06:04.622 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:06:04 smithi028 ceph-mon[23744]: from='mgr.14120 172.21.15.28:0/3751098848' entity='mgr.smithi028.azvyzy' 2024-09-16T08:06:06.273 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:06:06 smithi028 ceph-mon[23744]: from='client.? 172.21.15.28:0/3131457725' entity='client.admin' 2024-09-16T08:06:07.367 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:06:07 smithi028 ceph-mon[23744]: pgmap v3: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-16T08:06:07.367 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:06:07 smithi028 ceph-mon[23744]: from='client.? 172.21.15.28:0/683289157' entity='client.admin' 2024-09-16T08:06:07.369 INFO:teuthology.orchestra.run.smithi028.stderr:Enabling the dashboard module... 2024-09-16T08:06:08.446 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:06:08 smithi028 ceph-mon[23744]: from='client.? 172.21.15.28:0/2407868809' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "dashboard"}]: dispatch 2024-09-16T08:06:09.086 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:06:09 smithi028 ceph-mon[23744]: from='client.? 172.21.15.28:0/2407868809' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "dashboard"}]': finished 2024-09-16T08:06:09.086 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:06:09 smithi028 ceph-mon[23744]: mgrmap e9: smithi028.azvyzy(active, since 22s) 2024-09-16T08:06:09.548 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: { 2024-09-16T08:06:09.548 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: "epoch": 9, 2024-09-16T08:06:09.548 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: "available": true, 2024-09-16T08:06:09.548 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: "active_name": "smithi028.azvyzy", 2024-09-16T08:06:09.548 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: "num_standby": 0 2024-09-16T08:06:09.548 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: } 2024-09-16T08:06:10.157 INFO:teuthology.orchestra.run.smithi028.stderr:Waiting for the mgr to restart... 2024-09-16T08:06:10.158 INFO:teuthology.orchestra.run.smithi028.stderr:Waiting for mgr epoch 9... 2024-09-16T08:06:10.282 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:06:10 smithi028 ceph-mon[23744]: from='client.? 172.21.15.28:0/1714810069' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2024-09-16T08:06:15.526 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: { 2024-09-16T08:06:15.526 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: "mgrmap_epoch": 11, 2024-09-16T08:06:15.526 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: "initialized": true 2024-09-16T08:06:15.526 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: } 2024-09-16T08:06:15.797 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:06:15 smithi028 ceph-mon[23744]: Active manager daemon smithi028.azvyzy restarted 2024-09-16T08:06:15.797 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:06:15 smithi028 ceph-mon[23744]: Activating manager daemon smithi028.azvyzy 2024-09-16T08:06:15.798 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:06:15 smithi028 ceph-mon[23744]: osdmap e3: 0 total, 0 up, 0 in 2024-09-16T08:06:15.798 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:06:15 smithi028 ceph-mon[23744]: mgrmap e10: smithi028.azvyzy(active, starting, since 0.0527372s) 2024-09-16T08:06:15.798 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:06:15 smithi028 ceph-mon[23744]: from='mgr.14162 172.21.15.28:0/2716199755' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "mon metadata", "id": "smithi028"}]: dispatch 2024-09-16T08:06:15.798 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:06:15 smithi028 ceph-mon[23744]: from='mgr.14162 172.21.15.28:0/2716199755' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "mgr metadata", "who": "smithi028.azvyzy", "id": "smithi028.azvyzy"}]: dispatch 2024-09-16T08:06:15.798 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:06:15 smithi028 ceph-mon[23744]: from='mgr.14162 172.21.15.28:0/2716199755' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-16T08:06:15.798 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:06:15 smithi028 ceph-mon[23744]: from='mgr.14162 172.21.15.28:0/2716199755' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-16T08:06:15.798 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:06:15 smithi028 ceph-mon[23744]: from='mgr.14162 172.21.15.28:0/2716199755' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-16T08:06:15.798 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:06:15 smithi028 ceph-mon[23744]: Manager daemon smithi028.azvyzy is now available 2024-09-16T08:06:15.798 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:06:15 smithi028 ceph-mon[23744]: from='mgr.14162 172.21.15.28:0/2716199755' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-16T08:06:15.798 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:06:15 smithi028 ceph-mon[23744]: from='mgr.14162 172.21.15.28:0/2716199755' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-16T08:06:15.798 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:06:15 smithi028 ceph-mon[23744]: from='mgr.14162 172.21.15.28:0/2716199755' entity='mgr.smithi028.azvyzy' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi028.azvyzy/mirror_snapshot_schedule"}]: dispatch 2024-09-16T08:06:15.798 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:06:15 smithi028 ceph-mon[23744]: from='mgr.14162 172.21.15.28:0/2716199755' entity='mgr.smithi028.azvyzy' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi028.azvyzy/trash_purge_schedule"}]: dispatch 2024-09-16T08:06:15.798 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:06:15 smithi028 ceph-mon[23744]: [16/Sep/2024:08:06:14] ENGINE Bus STARTING 2024-09-16T08:06:15.798 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:06:15 smithi028 ceph-mon[23744]: [16/Sep/2024:08:06:15] ENGINE Serving on https://172.21.15.28:7150 2024-09-16T08:06:15.798 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:06:15 smithi028 ceph-mon[23744]: [16/Sep/2024:08:06:15] ENGINE Bus STARTED 2024-09-16T08:06:15.799 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:06:15 smithi028 ceph-mon[23744]: from='mgr.14162 172.21.15.28:0/2716199755' entity='mgr.smithi028.azvyzy' 2024-09-16T08:06:16.107 INFO:teuthology.orchestra.run.smithi028.stderr:mgr epoch 9 is available 2024-09-16T08:06:16.108 INFO:teuthology.orchestra.run.smithi028.stderr:Generating a dashboard self-signed certificate... 2024-09-16T08:06:16.679 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:06:16 smithi028 ceph-mon[23744]: from='client.14166 -' entity='client.admin' cmd=[{"prefix": "get_command_descriptions"}]: dispatch 2024-09-16T08:06:16.679 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:06:16 smithi028 ceph-mon[23744]: mgrmap e11: smithi028.azvyzy(active, since 1.05533s) 2024-09-16T08:06:16.679 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:06:16 smithi028 ceph-mon[23744]: from='client.14166 -' entity='client.admin' cmd=[{"prefix": "mgr_status"}]: dispatch 2024-09-16T08:06:16.679 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:06:16 smithi028 ceph-mon[23744]: from='mgr.14162 172.21.15.28:0/2716199755' entity='mgr.smithi028.azvyzy' 2024-09-16T08:06:16.816 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: Self-signed certificate created 2024-09-16T08:06:17.355 INFO:teuthology.orchestra.run.smithi028.stderr:Creating initial admin user... 2024-09-16T08:06:17.622 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:06:17 smithi028 ceph-mon[23744]: mgrmap e12: smithi028.azvyzy(active, since 2s) 2024-09-16T08:06:17.623 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:06:17 smithi028 ceph-mon[23744]: from='client.14174 -' entity='client.admin' cmd=[{"prefix": "dashboard create-self-signed-cert", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T08:06:17.623 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:06:17 smithi028 ceph-mon[23744]: from='mgr.14162 172.21.15.28:0/2716199755' entity='mgr.smithi028.azvyzy' 2024-09-16T08:06:17.623 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:06:17 smithi028 ceph-mon[23744]: from='mgr.14162 172.21.15.28:0/2716199755' entity='mgr.smithi028.azvyzy' 2024-09-16T08:06:17.623 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:06:17 smithi028 ceph-mon[23744]: from='mgr.14162 172.21.15.28:0/2716199755' entity='mgr.smithi028.azvyzy' 2024-09-16T08:06:18.275 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: {"username": "admin", "password": "$2b$12$VWMW14soSQ6zwpn6M5Mce.v2DJKM9xK95dLtGbJKvLeQ277oRplPC", "roles": ["administrator"], "name": null, "email": null, "lastUpdate": 1726473978, "enabled": true, "pwdExpirationDate": null, "pwdUpdateRequired": true} 2024-09-16T08:06:18.823 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:06:18 smithi028 ceph-mon[23744]: from='client.14176 -' entity='client.admin' cmd=[{"prefix": "dashboard ac-user-create", "username": "admin", "rolename": "administrator", "force_password": true, "pwd_update_required": true, "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T08:06:18.824 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:06:18 smithi028 ceph-mon[23744]: from='mgr.14162 172.21.15.28:0/2716199755' entity='mgr.smithi028.azvyzy' 2024-09-16T08:06:19.562 INFO:teuthology.orchestra.run.smithi028.stderr:Fetching dashboard port number... 2024-09-16T08:06:20.251 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: 8443 2024-09-16T08:06:20.837 INFO:teuthology.orchestra.run.smithi028.stderr:systemctl: disabled 2024-09-16T08:06:20.846 INFO:teuthology.orchestra.run.smithi028.stderr:systemctl: inactive 2024-09-16T08:06:20.846 INFO:teuthology.orchestra.run.smithi028.stderr:firewalld.service is not enabled 2024-09-16T08:06:20.846 INFO:teuthology.orchestra.run.smithi028.stderr:Not possible to open ports <[8443]>. firewalld.service is not available 2024-09-16T08:06:20.849 INFO:teuthology.orchestra.run.smithi028.stderr:Ceph Dashboard is now available at: 2024-09-16T08:06:20.849 INFO:teuthology.orchestra.run.smithi028.stderr: 2024-09-16T08:06:20.849 INFO:teuthology.orchestra.run.smithi028.stderr: URL: https://smithi028.front.sepia.ceph.com:8443/ 2024-09-16T08:06:20.849 INFO:teuthology.orchestra.run.smithi028.stderr: User: admin 2024-09-16T08:06:20.849 INFO:teuthology.orchestra.run.smithi028.stderr: Password: lankd8he9q 2024-09-16T08:06:20.849 INFO:teuthology.orchestra.run.smithi028.stderr: 2024-09-16T08:06:20.849 INFO:teuthology.orchestra.run.smithi028.stderr:Enabling autotune for osd_memory_target 2024-09-16T08:06:21.277 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:06:21 smithi028 ceph-mon[23744]: from='client.? 172.21.15.28:0/1833033061' entity='client.admin' cmd=[{"prefix": "config get", "who": "mgr", "key": "mgr/dashboard/ssl_server_port"}]: dispatch 2024-09-16T08:06:22.096 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:06:22 smithi028 ceph-mon[23744]: mgrmap e13: smithi028.azvyzy(active, since 6s) 2024-09-16T08:06:22.824 INFO:teuthology.orchestra.run.smithi028.stderr:/usr/bin/ceph: set mgr/dashboard/cluster/status 2024-09-16T08:06:23.455 INFO:teuthology.orchestra.run.smithi028.stderr:You can access the Ceph CLI with: 2024-09-16T08:06:23.456 INFO:teuthology.orchestra.run.smithi028.stderr: 2024-09-16T08:06:23.456 INFO:teuthology.orchestra.run.smithi028.stderr: sudo /home/ubuntu/cephtest/cephadm shell --fsid 5622b2fe-7402-11ef-bceb-c7b262605968 -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring 2024-09-16T08:06:23.456 INFO:teuthology.orchestra.run.smithi028.stderr: 2024-09-16T08:06:23.456 INFO:teuthology.orchestra.run.smithi028.stderr:Please consider enabling telemetry to help improve Ceph: 2024-09-16T08:06:23.456 INFO:teuthology.orchestra.run.smithi028.stderr: 2024-09-16T08:06:23.456 INFO:teuthology.orchestra.run.smithi028.stderr: ceph telemetry on 2024-09-16T08:06:23.457 INFO:teuthology.orchestra.run.smithi028.stderr: 2024-09-16T08:06:23.457 INFO:teuthology.orchestra.run.smithi028.stderr:For more information see: 2024-09-16T08:06:23.457 INFO:teuthology.orchestra.run.smithi028.stderr: 2024-09-16T08:06:23.457 INFO:teuthology.orchestra.run.smithi028.stderr: https://docs.ceph.com/docs/master/mgr/telemetry/ 2024-09-16T08:06:23.457 INFO:teuthology.orchestra.run.smithi028.stderr: 2024-09-16T08:06:23.457 INFO:teuthology.orchestra.run.smithi028.stderr:Bootstrap complete. 2024-09-16T08:06:23.491 INFO:tasks.cephadm:Fetching config... 2024-09-16T08:06:23.491 DEBUG:teuthology.orchestra.run.smithi028:> set -ex 2024-09-16T08:06:23.491 DEBUG:teuthology.orchestra.run.smithi028:> dd if=/etc/ceph/ceph.conf of=/dev/stdout 2024-09-16T08:06:23.509 INFO:tasks.cephadm:Fetching client.admin keyring... 2024-09-16T08:06:23.509 DEBUG:teuthology.orchestra.run.smithi028:> set -ex 2024-09-16T08:06:23.509 DEBUG:teuthology.orchestra.run.smithi028:> dd if=/etc/ceph/ceph.client.admin.keyring of=/dev/stdout 2024-09-16T08:06:23.566 INFO:tasks.cephadm:Fetching mon keyring... 2024-09-16T08:06:23.566 DEBUG:teuthology.orchestra.run.smithi028:> set -ex 2024-09-16T08:06:23.566 DEBUG:teuthology.orchestra.run.smithi028:> sudo dd if=/var/lib/ceph/5622b2fe-7402-11ef-bceb-c7b262605968/mon.smithi028/keyring of=/dev/stdout 2024-09-16T08:06:23.632 INFO:tasks.cephadm:Fetching pub ssh key... 2024-09-16T08:06:23.632 DEBUG:teuthology.orchestra.run.smithi028:> set -ex 2024-09-16T08:06:23.632 DEBUG:teuthology.orchestra.run.smithi028:> dd if=/home/ubuntu/cephtest/ceph.pub of=/dev/stdout 2024-09-16T08:06:23.688 INFO:tasks.cephadm:Installing pub ssh key for root users... 2024-09-16T08:06:23.689 DEBUG:teuthology.orchestra.run.smithi028:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCjplYqXUwyJsfnlJ6ByM+XAVT+9+Mb8iOLhsJmwBFz3irxfG5OSAMBrK9e6JkgwdH1YQvYmgNELlSTIZRwKYE6gd33FeASu5iEZbaHLDFuGJuBEuHmjOBxSX9bNo/BRPztTXBxzUvI+3CpzmJXqeB1+8fAbqR+BPH24cMNvygng0dvcbBAYT++7Nhwsv/7DjHf9CPrfSvdoNBZJbMa/o6mNsrkRnEW/7SKtlq1SBksXQN/mHRH8ADAqpyLk8qavDa6+DHSXKSGEvNIr9OxTZme946DaUZOai8RZPC1OWsSWBQ5p9rJgCaV4PT8+SN+NXXEIdH09ufDpuPMwfUz8GuWZo/1K9ihDXDsbsNdDEjgptRovpc22k0pGSBxJZnDODtTol2VB9MahAbFJvwhtl+brvwGEjUeRJ+Bf4BipYPN/ExoQnPnvlLpNf05aGO9Tzgl1ExlSzBijiCz+WmeojoMV6JdvuadoXASQmMLUVR4hlJvPfJNakSy6CyBpX1cNAc= ceph-5622b2fe-7402-11ef-bceb-c7b262605968' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2024-09-16T08:06:23.763 INFO:teuthology.orchestra.run.smithi028.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCjplYqXUwyJsfnlJ6ByM+XAVT+9+Mb8iOLhsJmwBFz3irxfG5OSAMBrK9e6JkgwdH1YQvYmgNELlSTIZRwKYE6gd33FeASu5iEZbaHLDFuGJuBEuHmjOBxSX9bNo/BRPztTXBxzUvI+3CpzmJXqeB1+8fAbqR+BPH24cMNvygng0dvcbBAYT++7Nhwsv/7DjHf9CPrfSvdoNBZJbMa/o6mNsrkRnEW/7SKtlq1SBksXQN/mHRH8ADAqpyLk8qavDa6+DHSXKSGEvNIr9OxTZme946DaUZOai8RZPC1OWsSWBQ5p9rJgCaV4PT8+SN+NXXEIdH09ufDpuPMwfUz8GuWZo/1K9ihDXDsbsNdDEjgptRovpc22k0pGSBxJZnDODtTol2VB9MahAbFJvwhtl+brvwGEjUeRJ+Bf4BipYPN/ExoQnPnvlLpNf05aGO9Tzgl1ExlSzBijiCz+WmeojoMV6JdvuadoXASQmMLUVR4hlJvPfJNakSy6CyBpX1cNAc= ceph-5622b2fe-7402-11ef-bceb-c7b262605968 2024-09-16T08:06:23.776 DEBUG:teuthology.orchestra.run.smithi144:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCjplYqXUwyJsfnlJ6ByM+XAVT+9+Mb8iOLhsJmwBFz3irxfG5OSAMBrK9e6JkgwdH1YQvYmgNELlSTIZRwKYE6gd33FeASu5iEZbaHLDFuGJuBEuHmjOBxSX9bNo/BRPztTXBxzUvI+3CpzmJXqeB1+8fAbqR+BPH24cMNvygng0dvcbBAYT++7Nhwsv/7DjHf9CPrfSvdoNBZJbMa/o6mNsrkRnEW/7SKtlq1SBksXQN/mHRH8ADAqpyLk8qavDa6+DHSXKSGEvNIr9OxTZme946DaUZOai8RZPC1OWsSWBQ5p9rJgCaV4PT8+SN+NXXEIdH09ufDpuPMwfUz8GuWZo/1K9ihDXDsbsNdDEjgptRovpc22k0pGSBxJZnDODtTol2VB9MahAbFJvwhtl+brvwGEjUeRJ+Bf4BipYPN/ExoQnPnvlLpNf05aGO9Tzgl1ExlSzBijiCz+WmeojoMV6JdvuadoXASQmMLUVR4hlJvPfJNakSy6CyBpX1cNAc= ceph-5622b2fe-7402-11ef-bceb-c7b262605968' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2024-09-16T08:06:23.832 INFO:teuthology.orchestra.run.smithi144.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCjplYqXUwyJsfnlJ6ByM+XAVT+9+Mb8iOLhsJmwBFz3irxfG5OSAMBrK9e6JkgwdH1YQvYmgNELlSTIZRwKYE6gd33FeASu5iEZbaHLDFuGJuBEuHmjOBxSX9bNo/BRPztTXBxzUvI+3CpzmJXqeB1+8fAbqR+BPH24cMNvygng0dvcbBAYT++7Nhwsv/7DjHf9CPrfSvdoNBZJbMa/o6mNsrkRnEW/7SKtlq1SBksXQN/mHRH8ADAqpyLk8qavDa6+DHSXKSGEvNIr9OxTZme946DaUZOai8RZPC1OWsSWBQ5p9rJgCaV4PT8+SN+NXXEIdH09ufDpuPMwfUz8GuWZo/1K9ihDXDsbsNdDEjgptRovpc22k0pGSBxJZnDODtTol2VB9MahAbFJvwhtl+brvwGEjUeRJ+Bf4BipYPN/ExoQnPnvlLpNf05aGO9Tzgl1ExlSzBijiCz+WmeojoMV6JdvuadoXASQmMLUVR4hlJvPfJNakSy6CyBpX1cNAc= ceph-5622b2fe-7402-11ef-bceb-c7b262605968 2024-09-16T08:06:23.849 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5622b2fe-7402-11ef-bceb-c7b262605968 -- ceph config set mgr mgr/cephadm/allow_ptrace true 2024-09-16T08:06:24.014 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:06:23 smithi028 ceph-mon[23744]: from='client.? 172.21.15.28:0/2802155491' entity='client.admin' 2024-09-16T08:06:26.279 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:06:26 smithi028 ceph-mon[23744]: from='mgr.14162 172.21.15.28:0/2716199755' entity='mgr.smithi028.azvyzy' 2024-09-16T08:06:26.279 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:06:26 smithi028 ceph-mon[23744]: from='mgr.14162 172.21.15.28:0/2716199755' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "config rm", "who": "osd/host:smithi028", "name": "osd_memory_target"}]: dispatch 2024-09-16T08:06:26.279 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:06:26 smithi028 ceph-mon[23744]: from='mgr.14162 172.21.15.28:0/2716199755' entity='mgr.smithi028.azvyzy' 2024-09-16T08:06:26.279 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:06:26 smithi028 ceph-mon[23744]: Deploying daemon alertmanager.smithi028 on smithi028 2024-09-16T08:06:27.211 INFO:tasks.cephadm:Distributing conf and client.admin keyring to all hosts + 0755 2024-09-16T08:06:27.212 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5622b2fe-7402-11ef-bceb-c7b262605968 -- ceph orch client-keyring set client.admin '*' --mode 0755 2024-09-16T08:06:27.787 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:06:27 smithi028 ceph-mon[23744]: from='client.? 172.21.15.28:0/3659767807' entity='client.admin' 2024-09-16T08:06:30.193 INFO:tasks.cephadm:Writing (initial) conf and keyring to smithi144 2024-09-16T08:06:30.193 DEBUG:teuthology.orchestra.run.smithi144:> set -ex 2024-09-16T08:06:30.193 DEBUG:teuthology.orchestra.run.smithi144:> dd of=/etc/ceph/ceph.conf 2024-09-16T08:06:30.212 DEBUG:teuthology.orchestra.run.smithi144:> set -ex 2024-09-16T08:06:30.212 DEBUG:teuthology.orchestra.run.smithi144:> dd of=/etc/ceph/ceph.client.admin.keyring 2024-09-16T08:06:30.271 INFO:tasks.cephadm:Adding host smithi144 to orchestrator... 2024-09-16T08:06:30.271 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5622b2fe-7402-11ef-bceb-c7b262605968 -- ceph orch host add smithi144 2024-09-16T08:06:30.446 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:06:30 smithi028 ceph-mon[23744]: from='client.14186 -' entity='client.admin' cmd=[{"prefix": "orch client-keyring set", "entity": "client.admin", "placement": "*", "mode": "0755", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T08:06:30.446 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:06:30 smithi028 ceph-mon[23744]: from='mgr.14162 172.21.15.28:0/2716199755' entity='mgr.smithi028.azvyzy' 2024-09-16T08:06:34.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:06:34 smithi028 ceph-mon[23744]: from='client.14188 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi144", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T08:06:34.616 INFO:teuthology.orchestra.run.smithi028.stdout:Added host 'smithi144' with addr '172.21.15.144' 2024-09-16T08:06:35.677 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:06:35 smithi028 ceph-mon[23744]: Deploying cephadm binary to smithi144 2024-09-16T08:06:35.677 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:06:35 smithi028 ceph-mon[23744]: pgmap v3: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-16T08:06:35.677 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:06:35 smithi028 ceph-mon[23744]: from='mgr.14162 172.21.15.28:0/2716199755' entity='mgr.smithi028.azvyzy' 2024-09-16T08:06:35.677 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:06:35 smithi028 ceph-mon[23744]: Added host smithi144 2024-09-16T08:06:35.704 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5622b2fe-7402-11ef-bceb-c7b262605968 -- ceph orch host ls --format=json 2024-09-16T08:06:37.816 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:06:37 smithi028 ceph-mon[23744]: pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-16T08:06:39.071 INFO:teuthology.orchestra.run.smithi028.stdout: 2024-09-16T08:06:39.072 INFO:teuthology.orchestra.run.smithi028.stdout:[{"addr": "172.21.15.28", "hostname": "smithi028", "labels": [], "status": ""}, {"addr": "172.21.15.144", "hostname": "smithi144", "labels": [], "status": ""}] 2024-09-16T08:06:39.696 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:06:39 smithi028 ceph-mon[23744]: from='mgr.14162 172.21.15.28:0/2716199755' entity='mgr.smithi028.azvyzy' 2024-09-16T08:06:39.697 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:06:39 smithi028 ceph-mon[23744]: from='mgr.14162 172.21.15.28:0/2716199755' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi028", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-09-16T08:06:39.697 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:06:39 smithi028 ceph-mon[23744]: from='mgr.14162 172.21.15.28:0/2716199755' entity='mgr.smithi028.azvyzy' cmd='[{"prefix": "auth get-or-create", "entity": "client.crash.smithi028", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]': finished 2024-09-16T08:06:39.697 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:06:39 smithi028 ceph-mon[23744]: from='mgr.14162 172.21.15.28:0/2716199755' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T08:06:39.697 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:06:39 smithi028 ceph-mon[23744]: Deploying daemon crash.smithi028 on smithi028 2024-09-16T08:06:39.697 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:06:39 smithi028 ceph-mon[23744]: pgmap v5: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-16T08:06:39.697 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:06:39 smithi028 ceph-mon[23744]: from='client.14190 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-16T08:06:40.782 INFO:tasks.cephadm:Setting crush tunables to default 2024-09-16T08:06:40.782 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5622b2fe-7402-11ef-bceb-c7b262605968 -- ceph osd crush tunables default 2024-09-16T08:06:40.808 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:06:40 smithi028 ceph-mon[23744]: from='mgr.14162 172.21.15.28:0/2716199755' entity='mgr.smithi028.azvyzy' 2024-09-16T08:06:42.331 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:06:42 smithi028 ceph-mon[23744]: pgmap v6: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-16T08:06:42.331 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:06:42 smithi028 ceph-mon[23744]: from='mgr.14162 172.21.15.28:0/2716199755' entity='mgr.smithi028.azvyzy' 2024-09-16T08:06:42.331 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:06:42 smithi028 ceph-mon[23744]: from='mgr.14162 172.21.15.28:0/2716199755' entity='mgr.smithi028.azvyzy' 2024-09-16T08:06:42.331 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:06:42 smithi028 ceph-mon[23744]: from='mgr.14162 172.21.15.28:0/2716199755' entity='mgr.smithi028.azvyzy' 2024-09-16T08:06:42.331 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:06:42 smithi028 ceph-mon[23744]: from='mgr.14162 172.21.15.28:0/2716199755' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-16T08:06:42.331 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:06:42 smithi028 ceph-mon[23744]: from='mgr.14162 172.21.15.28:0/2716199755' entity='mgr.smithi028.azvyzy' 2024-09-16T08:06:43.446 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:06:43 smithi028 ceph-mon[23744]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-16T08:06:43.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:06:43 smithi028 ceph-mon[23744]: Deploying daemon grafana.smithi028 on smithi028 2024-09-16T08:06:44.025 INFO:teuthology.orchestra.run.smithi028.stderr:adjusted tunables profile to default 2024-09-16T08:06:44.353 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:06:44 smithi028 ceph-mon[23744]: pgmap v7: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-16T08:06:44.354 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:06:44 smithi028 ceph-mon[23744]: from='client.? 172.21.15.28:0/3330293002' entity='client.admin' cmd=[{"prefix": "osd crush tunables", "profile": "default"}]: dispatch 2024-09-16T08:06:44.672 INFO:tasks.cephadm:Adding mon.smithi028 on smithi028 2024-09-16T08:06:44.672 INFO:tasks.cephadm:Adding mon.smithi144 on smithi144 2024-09-16T08:06:44.672 DEBUG:teuthology.orchestra.run.smithi144:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5622b2fe-7402-11ef-bceb-c7b262605968 -- ceph orch apply mon '2;smithi028:172.21.15.28=smithi028;smithi144:172.21.15.144=smithi144' 2024-09-16T08:06:45.446 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:06:45 smithi028 ceph-mon[23744]: from='client.? 172.21.15.28:0/3330293002' entity='client.admin' cmd='[{"prefix": "osd crush tunables", "profile": "default"}]': finished 2024-09-16T08:06:45.446 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:06:45 smithi028 ceph-mon[23744]: osdmap e4: 0 total, 0 up, 0 in 2024-09-16T08:06:45.446 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:06:45 smithi028 ceph-mon[23744]: from='mgr.14162 172.21.15.28:0/2716199755' entity='mgr.smithi028.azvyzy' 2024-09-16T08:06:46.317 INFO:teuthology.orchestra.run.smithi144.stdout:Scheduled mon update... 2024-09-16T08:06:46.446 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:06:46 smithi028 ceph-mon[23744]: pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-16T08:06:46.857 DEBUG:teuthology.orchestra.run.smithi144:mon.smithi144> sudo journalctl -f -n 0 -u ceph-5622b2fe-7402-11ef-bceb-c7b262605968@mon.smithi144.service 2024-09-16T08:06:46.859 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-16T08:06:46.859 DEBUG:teuthology.orchestra.run.smithi144:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5622b2fe-7402-11ef-bceb-c7b262605968 -- ceph mon dump -f json 2024-09-16T08:06:47.696 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:06:47 smithi028 ceph-mon[23744]: from='client.14194 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "placement": "2;smithi028:172.21.15.28=smithi028;smithi144:172.21.15.144=smithi144", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T08:06:47.696 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:06:47 smithi028 ceph-mon[23744]: Saving service mon spec with placement smithi028:172.21.15.28=smithi028;smithi144:172.21.15.144=smithi144;count:2 2024-09-16T08:06:47.697 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:06:47 smithi028 ceph-mon[23744]: from='mgr.14162 172.21.15.28:0/2716199755' entity='mgr.smithi028.azvyzy' 2024-09-16T08:06:47.697 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:06:47 smithi028 ceph-mon[23744]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-16T08:06:48.702 INFO:teuthology.orchestra.run.smithi144.stdout: 2024-09-16T08:06:48.702 INFO:teuthology.orchestra.run.smithi144.stdout:{"epoch":1,"fsid":"5622b2fe-7402-11ef-bceb-c7b262605968","modified":"2024-09-16T08:05:16.161732Z","created":"2024-09-16T08:05:16.161732Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi028","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:3300","nonce":0},{"type":"v1","addr":"172.21.15.28:6789","nonce":0}]},"addr":"172.21.15.28:6789/0","public_addr":"172.21.15.28:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-16T08:06:48.704 INFO:teuthology.orchestra.run.smithi144.stderr:dumped monmap epoch 1 2024-09-16T08:06:49.946 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:06:49 smithi028 ceph-mon[23744]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-16T08:06:49.946 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:06:49 smithi028 ceph-mon[23744]: from='client.? 172.21.15.144:0/3379275194' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-16T08:06:50.269 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-16T08:06:50.270 DEBUG:teuthology.orchestra.run.smithi144:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5622b2fe-7402-11ef-bceb-c7b262605968 -- ceph mon dump -f json 2024-09-16T08:06:52.079 INFO:teuthology.orchestra.run.smithi144.stdout: 2024-09-16T08:06:52.079 INFO:teuthology.orchestra.run.smithi144.stdout:{"epoch":1,"fsid":"5622b2fe-7402-11ef-bceb-c7b262605968","modified":"2024-09-16T08:05:16.161732Z","created":"2024-09-16T08:05:16.161732Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi028","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:3300","nonce":0},{"type":"v1","addr":"172.21.15.28:6789","nonce":0}]},"addr":"172.21.15.28:6789/0","public_addr":"172.21.15.28:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-16T08:06:52.080 INFO:teuthology.orchestra.run.smithi144.stderr:dumped monmap epoch 1 2024-09-16T08:06:52.269 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:06:52 smithi028 ceph-mon[23744]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-16T08:06:53.284 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:06:53 smithi028 ceph-mon[23744]: from='client.? 172.21.15.144:0/1181305665' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-16T08:06:53.638 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-16T08:06:53.639 DEBUG:teuthology.orchestra.run.smithi144:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5622b2fe-7402-11ef-bceb-c7b262605968 -- ceph mon dump -f json 2024-09-16T08:06:54.398 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:06:54 smithi028 ceph-mon[23744]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-16T08:06:54.399 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:06:54 smithi028 ceph-mon[23744]: from='mgr.14162 172.21.15.28:0/2716199755' entity='mgr.smithi028.azvyzy' 2024-09-16T08:06:55.309 INFO:teuthology.orchestra.run.smithi144.stdout: 2024-09-16T08:06:55.309 INFO:teuthology.orchestra.run.smithi144.stdout:{"epoch":1,"fsid":"5622b2fe-7402-11ef-bceb-c7b262605968","modified":"2024-09-16T08:05:16.161732Z","created":"2024-09-16T08:05:16.161732Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi028","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:3300","nonce":0},{"type":"v1","addr":"172.21.15.28:6789","nonce":0}]},"addr":"172.21.15.28:6789/0","public_addr":"172.21.15.28:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-16T08:06:55.311 INFO:teuthology.orchestra.run.smithi144.stderr:dumped monmap epoch 1 2024-09-16T08:06:55.403 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:06:55 smithi028 ceph-mon[23744]: Deploying daemon node-exporter.smithi028 on smithi028 2024-09-16T08:06:55.403 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:06:55 smithi028 ceph-mon[23744]: from='mgr.14162 172.21.15.28:0/2716199755' entity='mgr.smithi028.azvyzy' 2024-09-16T08:06:56.196 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:06:56 smithi028 ceph-mon[23744]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-16T08:06:56.196 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:06:56 smithi028 ceph-mon[23744]: from='client.? 172.21.15.144:0/479079119' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-16T08:06:56.877 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-16T08:06:56.878 DEBUG:teuthology.orchestra.run.smithi144:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5622b2fe-7402-11ef-bceb-c7b262605968 -- ceph mon dump -f json 2024-09-16T08:06:57.378 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:06:57 smithi028 ceph-mon[23744]: from='mgr.14162 172.21.15.28:0/2716199755' entity='mgr.smithi028.azvyzy' 2024-09-16T08:06:57.378 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:06:57 smithi028 ceph-mon[23744]: from='mgr.14162 172.21.15.28:0/2716199755' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2024-09-16T08:06:57.378 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:06:57 smithi028 ceph-mon[23744]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-16T08:06:58.362 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:06:58 smithi028 ceph-mon[23744]: from='mgr.14162 172.21.15.28:0/2716199755' entity='mgr.smithi028.azvyzy' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2024-09-16T08:06:58.362 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:06:58 smithi028 ceph-mon[23744]: mgrmap e14: smithi028.azvyzy(active, since 42s) 2024-09-16T08:06:58.675 INFO:teuthology.orchestra.run.smithi144.stdout: 2024-09-16T08:06:58.675 INFO:teuthology.orchestra.run.smithi144.stdout:{"epoch":1,"fsid":"5622b2fe-7402-11ef-bceb-c7b262605968","modified":"2024-09-16T08:05:16.161732Z","created":"2024-09-16T08:05:16.161732Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi028","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:3300","nonce":0},{"type":"v1","addr":"172.21.15.28:6789","nonce":0}]},"addr":"172.21.15.28:6789/0","public_addr":"172.21.15.28:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-16T08:06:58.677 INFO:teuthology.orchestra.run.smithi144.stderr:dumped monmap epoch 1 2024-09-16T08:06:59.446 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:06:59 smithi028 ceph-mon[23744]: from='client.? 172.21.15.144:0/641828129' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-16T08:07:00.269 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-16T08:07:00.270 DEBUG:teuthology.orchestra.run.smithi144:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5622b2fe-7402-11ef-bceb-c7b262605968 -- ceph mon dump -f json 2024-09-16T08:07:02.117 INFO:teuthology.orchestra.run.smithi144.stdout: 2024-09-16T08:07:02.117 INFO:teuthology.orchestra.run.smithi144.stdout:{"epoch":1,"fsid":"5622b2fe-7402-11ef-bceb-c7b262605968","modified":"2024-09-16T08:05:16.161732Z","created":"2024-09-16T08:05:16.161732Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi028","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:3300","nonce":0},{"type":"v1","addr":"172.21.15.28:6789","nonce":0}]},"addr":"172.21.15.28:6789/0","public_addr":"172.21.15.28:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-16T08:07:02.119 INFO:teuthology.orchestra.run.smithi144.stderr:dumped monmap epoch 1 2024-09-16T08:07:02.437 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:07:02 smithi028 ceph-mon[23744]: from='client.? 172.21.15.144:0/1774996331' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-16T08:07:03.674 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-16T08:07:03.674 DEBUG:teuthology.orchestra.run.smithi144:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5622b2fe-7402-11ef-bceb-c7b262605968 -- ceph mon dump -f json 2024-09-16T08:07:04.696 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:07:04 smithi028 ceph-mon[23744]: Active manager daemon smithi028.azvyzy restarted 2024-09-16T08:07:04.697 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:07:04 smithi028 ceph-mon[23744]: Activating manager daemon smithi028.azvyzy 2024-09-16T08:07:04.697 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:07:04 smithi028 ceph-mon[23744]: osdmap e5: 0 total, 0 up, 0 in 2024-09-16T08:07:04.697 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:07:04 smithi028 ceph-mon[23744]: mgrmap e15: smithi028.azvyzy(active, starting, since 0.0522253s) 2024-09-16T08:07:04.697 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:07:04 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "mon metadata", "id": "smithi028"}]: dispatch 2024-09-16T08:07:04.697 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:07:04 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "mgr metadata", "who": "smithi028.azvyzy", "id": "smithi028.azvyzy"}]: dispatch 2024-09-16T08:07:04.697 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:07:04 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-16T08:07:04.697 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:07:04 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-16T08:07:04.697 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:07:04 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-16T08:07:04.697 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:07:04 smithi028 ceph-mon[23744]: Manager daemon smithi028.azvyzy is now available 2024-09-16T08:07:04.697 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:07:04 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-16T08:07:04.698 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:07:04 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-16T08:07:04.698 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:07:04 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T08:07:04.698 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:07:04 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-16T08:07:04.698 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:07:04 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' 2024-09-16T08:07:04.698 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:07:04 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi028.azvyzy/mirror_snapshot_schedule"}]: dispatch 2024-09-16T08:07:04.698 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:07:04 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi028.azvyzy/trash_purge_schedule"}]: dispatch 2024-09-16T08:07:04.698 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:07:04 smithi028 ceph-mon[23744]: [16/Sep/2024:08:07:03] ENGINE Bus STARTING 2024-09-16T08:07:04.698 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:07:04 smithi028 ceph-mon[23744]: [16/Sep/2024:08:07:04] ENGINE Serving on https://172.21.15.28:7150 2024-09-16T08:07:04.698 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:07:04 smithi028 ceph-mon[23744]: [16/Sep/2024:08:07:04] ENGINE Bus STARTED 2024-09-16T08:07:04.698 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:07:04 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' 2024-09-16T08:07:04.698 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:07:04 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' 2024-09-16T08:07:05.622 INFO:teuthology.orchestra.run.smithi144.stdout: 2024-09-16T08:07:05.623 INFO:teuthology.orchestra.run.smithi144.stdout:{"epoch":1,"fsid":"5622b2fe-7402-11ef-bceb-c7b262605968","modified":"2024-09-16T08:05:16.161732Z","created":"2024-09-16T08:05:16.161732Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi028","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:3300","nonce":0},{"type":"v1","addr":"172.21.15.28:6789","nonce":0}]},"addr":"172.21.15.28:6789/0","public_addr":"172.21.15.28:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-16T08:07:05.624 INFO:teuthology.orchestra.run.smithi144.stderr:dumped monmap epoch 1 2024-09-16T08:07:05.862 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:07:05 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' 2024-09-16T08:07:05.863 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:07:05 smithi028 ceph-mon[23744]: mgrmap e16: smithi028.azvyzy(active, since 1.05471s) 2024-09-16T08:07:05.863 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:07:05 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' 2024-09-16T08:07:06.696 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:07:06 smithi028 ceph-mon[23744]: from='client.? 172.21.15.144:0/2563689989' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-16T08:07:06.696 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:07:06 smithi028 ceph-mon[23744]: mgrmap e17: smithi028.azvyzy(active, since 2s) 2024-09-16T08:07:07.179 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-16T08:07:07.179 DEBUG:teuthology.orchestra.run.smithi144:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5622b2fe-7402-11ef-bceb-c7b262605968 -- ceph mon dump -f json 2024-09-16T08:07:08.196 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:07:08 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' 2024-09-16T08:07:08.196 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:07:08 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' 2024-09-16T08:07:08.196 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:07:08 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "config rm", "who": "osd/host:smithi028", "name": "osd_memory_target"}]: dispatch 2024-09-16T08:07:08.196 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:07:08 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' 2024-09-16T08:07:09.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:07:09 smithi028 ceph-mon[23744]: Updating smithi028:/etc/ceph/ceph.conf 2024-09-16T08:07:09.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:07:09 smithi028 ceph-mon[23744]: Updating smithi028:/etc/ceph/ceph.client.admin.keyring 2024-09-16T08:07:09.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:07:09 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' 2024-09-16T08:07:09.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:07:09 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' 2024-09-16T08:07:09.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:07:09 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "config rm", "who": "osd/host:smithi144", "name": "osd_memory_target"}]: dispatch 2024-09-16T08:07:09.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:07:09 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' 2024-09-16T08:07:09.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:07:09 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi144", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-09-16T08:07:09.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:07:09 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd='[{"prefix": "auth get-or-create", "entity": "client.crash.smithi144", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]': finished 2024-09-16T08:07:09.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:07:09 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T08:07:09.917 INFO:teuthology.orchestra.run.smithi144.stdout: 2024-09-16T08:07:09.917 INFO:teuthology.orchestra.run.smithi144.stdout:{"epoch":1,"fsid":"5622b2fe-7402-11ef-bceb-c7b262605968","modified":"2024-09-16T08:05:16.161732Z","created":"2024-09-16T08:05:16.161732Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi028","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:3300","nonce":0},{"type":"v1","addr":"172.21.15.28:6789","nonce":0}]},"addr":"172.21.15.28:6789/0","public_addr":"172.21.15.28:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-16T08:07:09.919 INFO:teuthology.orchestra.run.smithi144.stderr:dumped monmap epoch 1 2024-09-16T08:07:10.446 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:07:10 smithi028 ceph-mon[23744]: Deploying daemon crash.smithi144 on smithi144 2024-09-16T08:07:10.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:07:10 smithi028 ceph-mon[23744]: from='client.? 172.21.15.144:0/3526189570' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-16T08:07:11.799 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-16T08:07:11.799 DEBUG:teuthology.orchestra.run.smithi144:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5622b2fe-7402-11ef-bceb-c7b262605968 -- ceph mon dump -f json 2024-09-16T08:07:13.946 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:07:13 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' 2024-09-16T08:07:13.947 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:07:13 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi144.utzszj", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-16T08:07:13.947 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:07:13 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.smithi144.utzszj", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2024-09-16T08:07:13.947 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:07:13 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-16T08:07:13.947 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:07:13 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T08:07:13.947 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:07:13 smithi028 ceph-mon[23744]: Deploying daemon mgr.smithi144.utzszj on smithi144 2024-09-16T08:07:14.946 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:07:14 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' 2024-09-16T08:07:15.320 INFO:teuthology.orchestra.run.smithi144.stdout: 2024-09-16T08:07:15.321 INFO:teuthology.orchestra.run.smithi144.stdout:{"epoch":1,"fsid":"5622b2fe-7402-11ef-bceb-c7b262605968","modified":"2024-09-16T08:05:16.161732Z","created":"2024-09-16T08:05:16.161732Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi028","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:3300","nonce":0},{"type":"v1","addr":"172.21.15.28:6789","nonce":0}]},"addr":"172.21.15.28:6789/0","public_addr":"172.21.15.28:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-16T08:07:15.322 INFO:teuthology.orchestra.run.smithi144.stderr:dumped monmap epoch 1 2024-09-16T08:07:16.446 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:07:16 smithi028 ceph-mon[23744]: from='client.? 172.21.15.144:0/2356483929' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-16T08:07:16.446 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:07:16 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' 2024-09-16T08:07:16.446 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:07:16 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-16T08:07:16.446 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:07:16 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T08:07:17.078 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-16T08:07:17.079 DEBUG:teuthology.orchestra.run.smithi144:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5622b2fe-7402-11ef-bceb-c7b262605968 -- ceph mon dump -f json 2024-09-16T08:07:17.446 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:07:17 smithi028 ceph-mon[23744]: Deploying daemon mon.smithi144 on smithi144 2024-09-16T08:07:19.948 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:07:19 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' 2024-09-16T08:07:20.090 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:07:20 smithi144 systemd[1]: Started Ceph mon.smithi144 for 5622b2fe-7402-11ef-bceb-c7b262605968. 2024-09-16T08:07:25.036 INFO:teuthology.orchestra.run.smithi144.stdout: 2024-09-16T08:07:25.036 INFO:teuthology.orchestra.run.smithi144.stdout:{"epoch":2,"fsid":"5622b2fe-7402-11ef-bceb-c7b262605968","modified":"2024-09-16T08:07:20.025003Z","created":"2024-09-16T08:05:16.161732Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi028","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:3300","nonce":0},{"type":"v1","addr":"172.21.15.28:6789","nonce":0}]},"addr":"172.21.15.28:6789/0","public_addr":"172.21.15.28:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":1,"name":"smithi144","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:3300","nonce":0},{"type":"v1","addr":"172.21.15.144:6789","nonce":0}]},"addr":"172.21.15.144:6789/0","public_addr":"172.21.15.144:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0,1]} 2024-09-16T08:07:25.037 INFO:teuthology.orchestra.run.smithi144.stderr:dumped monmap epoch 2 2024-09-16T08:07:25.308 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:07:25 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "mon metadata", "id": "smithi028"}]: dispatch 2024-09-16T08:07:25.308 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:07:25 smithi144 ceph-mon[25668]: mon.smithi028 calling monitor election 2024-09-16T08:07:25.308 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:07:25 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "mon metadata", "id": "smithi144"}]: dispatch 2024-09-16T08:07:25.308 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:07:25 smithi144 ceph-mon[25668]: from='client.? 172.21.15.144:0/192199418' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-16T08:07:25.308 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:07:25 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "mon metadata", "id": "smithi144"}]: dispatch 2024-09-16T08:07:25.309 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:07:25 smithi144 ceph-mon[25668]: from='mgr.? 172.21.15.144:0/2685061706' entity='mgr.smithi144.utzszj' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi144.utzszj/crt"}]: dispatch 2024-09-16T08:07:25.309 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:07:25 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "mon metadata", "id": "smithi144"}]: dispatch 2024-09-16T08:07:25.309 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:07:25 smithi144 ceph-mon[25668]: mon.smithi144 calling monitor election 2024-09-16T08:07:25.309 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:07:25 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "mon metadata", "id": "smithi144"}]: dispatch 2024-09-16T08:07:25.309 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:07:25 smithi144 ceph-mon[25668]: pgmap v3: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-16T08:07:25.309 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:07:25 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "mon metadata", "id": "smithi144"}]: dispatch 2024-09-16T08:07:25.309 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:07:25 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "mon metadata", "id": "smithi144"}]: dispatch 2024-09-16T08:07:25.309 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:07:25 smithi144 ceph-mon[25668]: mon.smithi028 is new leader, mons smithi028,smithi144 in quorum (ranks 0,1) 2024-09-16T08:07:25.309 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:07:25 smithi144 ceph-mon[25668]: monmap e2: 2 mons at {smithi028=[v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0],smithi144=[v2:172.21.15.144:3300/0,v1:172.21.15.144:6789/0]} 2024-09-16T08:07:25.309 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:07:25 smithi144 ceph-mon[25668]: fsmap 2024-09-16T08:07:25.309 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:07:25 smithi144 ceph-mon[25668]: osdmap e5: 0 total, 0 up, 0 in 2024-09-16T08:07:25.309 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:07:25 smithi144 ceph-mon[25668]: mgrmap e17: smithi028.azvyzy(active, since 21s) 2024-09-16T08:07:25.309 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:07:25 smithi144 ceph-mon[25668]: Standby manager daemon smithi144.utzszj started 2024-09-16T08:07:25.309 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:07:25 smithi144 ceph-mon[25668]: from='mgr.? 172.21.15.144:0/2685061706' entity='mgr.smithi144.utzszj' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-16T08:07:25.310 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:07:25 smithi144 ceph-mon[25668]: overall HEALTH_OK 2024-09-16T08:07:25.310 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:07:25 smithi144 ceph-mon[25668]: from='mgr.? 172.21.15.144:0/2685061706' entity='mgr.smithi144.utzszj' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi144.utzszj/key"}]: dispatch 2024-09-16T08:07:25.310 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:07:25 smithi144 ceph-mon[25668]: from='mgr.? 172.21.15.144:0/2685061706' entity='mgr.smithi144.utzszj' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-16T08:07:25.310 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:07:25 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' 2024-09-16T08:07:25.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:07:25 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "mon metadata", "id": "smithi028"}]: dispatch 2024-09-16T08:07:25.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:07:25 smithi028 ceph-mon[23744]: mon.smithi028 calling monitor election 2024-09-16T08:07:25.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:07:25 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "mon metadata", "id": "smithi144"}]: dispatch 2024-09-16T08:07:25.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:07:25 smithi028 ceph-mon[23744]: from='client.? 172.21.15.144:0/192199418' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-16T08:07:25.448 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:07:25 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "mon metadata", "id": "smithi144"}]: dispatch 2024-09-16T08:07:25.448 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:07:25 smithi028 ceph-mon[23744]: from='mgr.? 172.21.15.144:0/2685061706' entity='mgr.smithi144.utzszj' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi144.utzszj/crt"}]: dispatch 2024-09-16T08:07:25.448 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:07:25 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "mon metadata", "id": "smithi144"}]: dispatch 2024-09-16T08:07:25.448 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:07:25 smithi028 ceph-mon[23744]: mon.smithi144 calling monitor election 2024-09-16T08:07:25.448 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:07:25 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "mon metadata", "id": "smithi144"}]: dispatch 2024-09-16T08:07:25.448 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:07:25 smithi028 ceph-mon[23744]: pgmap v3: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-16T08:07:25.449 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:07:25 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "mon metadata", "id": "smithi144"}]: dispatch 2024-09-16T08:07:25.449 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:07:25 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "mon metadata", "id": "smithi144"}]: dispatch 2024-09-16T08:07:25.449 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:07:25 smithi028 ceph-mon[23744]: mon.smithi028 is new leader, mons smithi028,smithi144 in quorum (ranks 0,1) 2024-09-16T08:07:25.449 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:07:25 smithi028 ceph-mon[23744]: monmap e2: 2 mons at {smithi028=[v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0],smithi144=[v2:172.21.15.144:3300/0,v1:172.21.15.144:6789/0]} 2024-09-16T08:07:25.449 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:07:25 smithi028 ceph-mon[23744]: fsmap 2024-09-16T08:07:25.449 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:07:25 smithi028 ceph-mon[23744]: osdmap e5: 0 total, 0 up, 0 in 2024-09-16T08:07:25.450 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:07:25 smithi028 ceph-mon[23744]: mgrmap e17: smithi028.azvyzy(active, since 21s) 2024-09-16T08:07:25.450 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:07:25 smithi028 ceph-mon[23744]: Standby manager daemon smithi144.utzszj started 2024-09-16T08:07:25.450 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:07:25 smithi028 ceph-mon[23744]: from='mgr.? 172.21.15.144:0/2685061706' entity='mgr.smithi144.utzszj' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-16T08:07:25.450 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:07:25 smithi028 ceph-mon[23744]: overall HEALTH_OK 2024-09-16T08:07:25.450 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:07:25 smithi028 ceph-mon[23744]: from='mgr.? 172.21.15.144:0/2685061706' entity='mgr.smithi144.utzszj' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi144.utzszj/key"}]: dispatch 2024-09-16T08:07:25.450 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:07:25 smithi028 ceph-mon[23744]: from='mgr.? 172.21.15.144:0/2685061706' entity='mgr.smithi144.utzszj' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-16T08:07:25.450 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:07:25 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' 2024-09-16T08:07:25.632 INFO:tasks.cephadm:Generating final ceph.conf file... 2024-09-16T08:07:25.632 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5622b2fe-7402-11ef-bceb-c7b262605968 -- ceph config generate-minimal-conf 2024-09-16T08:07:26.196 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:07:26 smithi028 ceph-mon[23744]: Deploying daemon node-exporter.smithi144 on smithi144 2024-09-16T08:07:26.197 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:07:26 smithi028 ceph-mon[23744]: mgrmap e18: smithi028.azvyzy(active, since 21s), standbys: smithi144.utzszj 2024-09-16T08:07:26.197 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:07:26 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "mgr metadata", "who": "smithi144.utzszj", "id": "smithi144.utzszj"}]: dispatch 2024-09-16T08:07:26.197 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:07:26 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "mon metadata", "id": "smithi144"}]: dispatch 2024-09-16T08:07:26.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:07:26 smithi144 ceph-mon[25668]: Deploying daemon node-exporter.smithi144 on smithi144 2024-09-16T08:07:26.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:07:26 smithi144 ceph-mon[25668]: mgrmap e18: smithi028.azvyzy(active, since 21s), standbys: smithi144.utzszj 2024-09-16T08:07:26.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:07:26 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "mgr metadata", "who": "smithi144.utzszj", "id": "smithi144.utzszj"}]: dispatch 2024-09-16T08:07:26.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:07:26 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "mon metadata", "id": "smithi144"}]: dispatch 2024-09-16T08:07:27.446 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:07:27 smithi028 ceph-mon[23744]: pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-16T08:07:27.476 INFO:teuthology.orchestra.run.smithi028.stdout:# minimal ceph.conf for 5622b2fe-7402-11ef-bceb-c7b262605968 2024-09-16T08:07:27.476 INFO:teuthology.orchestra.run.smithi028.stdout:[global] 2024-09-16T08:07:27.476 INFO:teuthology.orchestra.run.smithi028.stdout: fsid = 5622b2fe-7402-11ef-bceb-c7b262605968 2024-09-16T08:07:27.477 INFO:teuthology.orchestra.run.smithi028.stdout: mon_host = [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] [v2:172.21.15.144:3300/0,v1:172.21.15.144:6789/0] 2024-09-16T08:07:27.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:07:27 smithi144 ceph-mon[25668]: pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-16T08:07:28.070 INFO:tasks.cephadm:Distributing (final) config and client.admin keyring... 2024-09-16T08:07:28.071 DEBUG:teuthology.orchestra.run.smithi028:> set -ex 2024-09-16T08:07:28.071 DEBUG:teuthology.orchestra.run.smithi028:> sudo dd of=/etc/ceph/ceph.conf 2024-09-16T08:07:28.103 DEBUG:teuthology.orchestra.run.smithi028:> set -ex 2024-09-16T08:07:28.103 DEBUG:teuthology.orchestra.run.smithi028:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2024-09-16T08:07:28.170 DEBUG:teuthology.orchestra.run.smithi144:> set -ex 2024-09-16T08:07:28.170 DEBUG:teuthology.orchestra.run.smithi144:> sudo dd of=/etc/ceph/ceph.conf 2024-09-16T08:07:28.199 DEBUG:teuthology.orchestra.run.smithi144:> set -ex 2024-09-16T08:07:28.199 DEBUG:teuthology.orchestra.run.smithi144:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2024-09-16T08:07:28.264 DEBUG:tasks.cephadm:set 0 configs 2024-09-16T08:07:28.264 INFO:tasks.cephadm:Deploying OSDs... 2024-09-16T08:07:28.265 DEBUG:teuthology.orchestra.run.smithi028:> set -ex 2024-09-16T08:07:28.265 DEBUG:teuthology.orchestra.run.smithi028:> dd if=/scratch_devs of=/dev/stdout 2024-09-16T08:07:28.281 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2024-09-16T08:07:28.281 DEBUG:teuthology.orchestra.run.smithi028:> stat /dev/vg_nvme/lv_1 2024-09-16T08:07:28.321 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:07:28 smithi028 ceph-mon[23744]: from='client.? 172.21.15.28:0/2163490343' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T08:07:28.336 INFO:teuthology.orchestra.run.smithi028.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2024-09-16T08:07:28.336 INFO:teuthology.orchestra.run.smithi028.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-16T08:07:28.337 INFO:teuthology.orchestra.run.smithi028.stdout:Device: 5h/5d Inode: 665 Links: 1 2024-09-16T08:07:28.337 INFO:teuthology.orchestra.run.smithi028.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-16T08:07:28.337 INFO:teuthology.orchestra.run.smithi028.stdout:Context: system_u:object_r:device_t:s0 2024-09-16T08:07:28.337 INFO:teuthology.orchestra.run.smithi028.stdout:Access: 2024-09-16 08:06:53.810861039 +0000 2024-09-16T08:07:28.337 INFO:teuthology.orchestra.run.smithi028.stdout:Modify: 2024-09-16 08:04:26.406310396 +0000 2024-09-16T08:07:28.337 INFO:teuthology.orchestra.run.smithi028.stdout:Change: 2024-09-16 08:04:26.406310396 +0000 2024-09-16T08:07:28.337 INFO:teuthology.orchestra.run.smithi028.stdout: Birth: 2024-09-16 08:04:26.406310396 +0000 2024-09-16T08:07:28.337 DEBUG:teuthology.orchestra.run.smithi028:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2024-09-16T08:07:28.401 INFO:teuthology.orchestra.run.smithi028.stderr:1+0 records in 2024-09-16T08:07:28.401 INFO:teuthology.orchestra.run.smithi028.stderr:1+0 records out 2024-09-16T08:07:28.401 INFO:teuthology.orchestra.run.smithi028.stderr:512 bytes copied, 0.000131518 s, 3.9 MB/s 2024-09-16T08:07:28.402 DEBUG:teuthology.orchestra.run.smithi028:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2024-09-16T08:07:28.461 DEBUG:teuthology.orchestra.run.smithi028:> stat /dev/vg_nvme/lv_2 2024-09-16T08:07:28.518 INFO:teuthology.orchestra.run.smithi028.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2024-09-16T08:07:28.518 INFO:teuthology.orchestra.run.smithi028.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-16T08:07:28.519 INFO:teuthology.orchestra.run.smithi028.stdout:Device: 5h/5d Inode: 684 Links: 1 2024-09-16T08:07:28.519 INFO:teuthology.orchestra.run.smithi028.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-16T08:07:28.519 INFO:teuthology.orchestra.run.smithi028.stdout:Context: system_u:object_r:device_t:s0 2024-09-16T08:07:28.519 INFO:teuthology.orchestra.run.smithi028.stdout:Access: 2024-09-16 08:06:53.810861039 +0000 2024-09-16T08:07:28.519 INFO:teuthology.orchestra.run.smithi028.stdout:Modify: 2024-09-16 08:04:26.723301793 +0000 2024-09-16T08:07:28.519 INFO:teuthology.orchestra.run.smithi028.stdout:Change: 2024-09-16 08:04:26.723301793 +0000 2024-09-16T08:07:28.519 INFO:teuthology.orchestra.run.smithi028.stdout: Birth: 2024-09-16 08:04:26.723301793 +0000 2024-09-16T08:07:28.519 DEBUG:teuthology.orchestra.run.smithi028:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2024-09-16T08:07:28.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:07:28 smithi144 ceph-mon[25668]: from='client.? 172.21.15.28:0/2163490343' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T08:07:28.584 INFO:teuthology.orchestra.run.smithi028.stderr:1+0 records in 2024-09-16T08:07:28.584 INFO:teuthology.orchestra.run.smithi028.stderr:1+0 records out 2024-09-16T08:07:28.585 INFO:teuthology.orchestra.run.smithi028.stderr:512 bytes copied, 0.000118269 s, 4.3 MB/s 2024-09-16T08:07:28.586 DEBUG:teuthology.orchestra.run.smithi028:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2024-09-16T08:07:28.643 DEBUG:teuthology.orchestra.run.smithi028:> stat /dev/vg_nvme/lv_3 2024-09-16T08:07:28.699 INFO:teuthology.orchestra.run.smithi028.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2024-09-16T08:07:28.699 INFO:teuthology.orchestra.run.smithi028.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-16T08:07:28.699 INFO:teuthology.orchestra.run.smithi028.stdout:Device: 5h/5d Inode: 703 Links: 1 2024-09-16T08:07:28.699 INFO:teuthology.orchestra.run.smithi028.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-16T08:07:28.699 INFO:teuthology.orchestra.run.smithi028.stdout:Context: system_u:object_r:device_t:s0 2024-09-16T08:07:28.699 INFO:teuthology.orchestra.run.smithi028.stdout:Access: 2024-09-16 08:06:53.810861039 +0000 2024-09-16T08:07:28.699 INFO:teuthology.orchestra.run.smithi028.stdout:Modify: 2024-09-16 08:04:27.056292756 +0000 2024-09-16T08:07:28.700 INFO:teuthology.orchestra.run.smithi028.stdout:Change: 2024-09-16 08:04:27.056292756 +0000 2024-09-16T08:07:28.700 INFO:teuthology.orchestra.run.smithi028.stdout: Birth: 2024-09-16 08:04:27.056292756 +0000 2024-09-16T08:07:28.700 DEBUG:teuthology.orchestra.run.smithi028:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2024-09-16T08:07:28.764 INFO:teuthology.orchestra.run.smithi028.stderr:1+0 records in 2024-09-16T08:07:28.764 INFO:teuthology.orchestra.run.smithi028.stderr:1+0 records out 2024-09-16T08:07:28.764 INFO:teuthology.orchestra.run.smithi028.stderr:512 bytes copied, 0.000108972 s, 4.7 MB/s 2024-09-16T08:07:28.765 DEBUG:teuthology.orchestra.run.smithi028:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2024-09-16T08:07:28.821 DEBUG:teuthology.orchestra.run.smithi028:> stat /dev/vg_nvme/lv_4 2024-09-16T08:07:28.876 INFO:teuthology.orchestra.run.smithi028.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2024-09-16T08:07:28.876 INFO:teuthology.orchestra.run.smithi028.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-16T08:07:28.877 INFO:teuthology.orchestra.run.smithi028.stdout:Device: 5h/5d Inode: 725 Links: 1 2024-09-16T08:07:28.877 INFO:teuthology.orchestra.run.smithi028.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-16T08:07:28.877 INFO:teuthology.orchestra.run.smithi028.stdout:Context: system_u:object_r:device_t:s0 2024-09-16T08:07:28.877 INFO:teuthology.orchestra.run.smithi028.stdout:Access: 2024-09-16 08:06:53.811861012 +0000 2024-09-16T08:07:28.877 INFO:teuthology.orchestra.run.smithi028.stdout:Modify: 2024-09-16 08:04:27.377284044 +0000 2024-09-16T08:07:28.877 INFO:teuthology.orchestra.run.smithi028.stdout:Change: 2024-09-16 08:04:27.377284044 +0000 2024-09-16T08:07:28.877 INFO:teuthology.orchestra.run.smithi028.stdout: Birth: 2024-09-16 08:04:27.377284044 +0000 2024-09-16T08:07:28.877 DEBUG:teuthology.orchestra.run.smithi028:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2024-09-16T08:07:28.941 INFO:teuthology.orchestra.run.smithi028.stderr:1+0 records in 2024-09-16T08:07:28.941 INFO:teuthology.orchestra.run.smithi028.stderr:1+0 records out 2024-09-16T08:07:28.941 INFO:teuthology.orchestra.run.smithi028.stderr:512 bytes copied, 0.000102469 s, 5.0 MB/s 2024-09-16T08:07:28.942 DEBUG:teuthology.orchestra.run.smithi028:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2024-09-16T08:07:28.999 DEBUG:teuthology.orchestra.run.smithi144:> set -ex 2024-09-16T08:07:28.999 DEBUG:teuthology.orchestra.run.smithi144:> dd if=/scratch_devs of=/dev/stdout 2024-09-16T08:07:29.015 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2024-09-16T08:07:29.015 DEBUG:teuthology.orchestra.run.smithi144:> stat /dev/vg_nvme/lv_1 2024-09-16T08:07:29.070 INFO:teuthology.orchestra.run.smithi144.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2024-09-16T08:07:29.070 INFO:teuthology.orchestra.run.smithi144.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-16T08:07:29.070 INFO:teuthology.orchestra.run.smithi144.stdout:Device: 5h/5d Inode: 667 Links: 1 2024-09-16T08:07:29.070 INFO:teuthology.orchestra.run.smithi144.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-16T08:07:29.070 INFO:teuthology.orchestra.run.smithi144.stdout:Context: system_u:object_r:device_t:s0 2024-09-16T08:07:29.070 INFO:teuthology.orchestra.run.smithi144.stdout:Access: 2024-09-16 08:07:25.939041039 +0000 2024-09-16T08:07:29.070 INFO:teuthology.orchestra.run.smithi144.stdout:Modify: 2024-09-16 08:04:24.170718010 +0000 2024-09-16T08:07:29.071 INFO:teuthology.orchestra.run.smithi144.stdout:Change: 2024-09-16 08:04:24.170718010 +0000 2024-09-16T08:07:29.071 INFO:teuthology.orchestra.run.smithi144.stdout: Birth: 2024-09-16 08:04:24.170718010 +0000 2024-09-16T08:07:29.071 DEBUG:teuthology.orchestra.run.smithi144:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2024-09-16T08:07:29.134 INFO:teuthology.orchestra.run.smithi144.stderr:1+0 records in 2024-09-16T08:07:29.134 INFO:teuthology.orchestra.run.smithi144.stderr:1+0 records out 2024-09-16T08:07:29.134 INFO:teuthology.orchestra.run.smithi144.stderr:512 bytes copied, 0.00019511 s, 2.6 MB/s 2024-09-16T08:07:29.135 DEBUG:teuthology.orchestra.run.smithi144:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2024-09-16T08:07:29.191 DEBUG:teuthology.orchestra.run.smithi144:> stat /dev/vg_nvme/lv_2 2024-09-16T08:07:29.196 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:07:29 smithi028 ceph-mon[23744]: pgmap v5: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-16T08:07:29.196 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:07:29 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' 2024-09-16T08:07:29.196 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:07:29 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' 2024-09-16T08:07:29.246 INFO:teuthology.orchestra.run.smithi144.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2024-09-16T08:07:29.246 INFO:teuthology.orchestra.run.smithi144.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-16T08:07:29.246 INFO:teuthology.orchestra.run.smithi144.stdout:Device: 5h/5d Inode: 687 Links: 1 2024-09-16T08:07:29.246 INFO:teuthology.orchestra.run.smithi144.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-16T08:07:29.246 INFO:teuthology.orchestra.run.smithi144.stdout:Context: system_u:object_r:device_t:s0 2024-09-16T08:07:29.246 INFO:teuthology.orchestra.run.smithi144.stdout:Access: 2024-09-16 08:07:25.939041039 +0000 2024-09-16T08:07:29.247 INFO:teuthology.orchestra.run.smithi144.stdout:Modify: 2024-09-16 08:04:24.501712372 +0000 2024-09-16T08:07:29.247 INFO:teuthology.orchestra.run.smithi144.stdout:Change: 2024-09-16 08:04:24.501712372 +0000 2024-09-16T08:07:29.247 INFO:teuthology.orchestra.run.smithi144.stdout: Birth: 2024-09-16 08:04:24.501712372 +0000 2024-09-16T08:07:29.247 DEBUG:teuthology.orchestra.run.smithi144:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2024-09-16T08:07:29.307 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:07:29 smithi144 ceph-mon[25668]: pgmap v5: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-16T08:07:29.307 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:07:29 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' 2024-09-16T08:07:29.307 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:07:29 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' 2024-09-16T08:07:29.309 INFO:teuthology.orchestra.run.smithi144.stderr:1+0 records in 2024-09-16T08:07:29.309 INFO:teuthology.orchestra.run.smithi144.stderr:1+0 records out 2024-09-16T08:07:29.309 INFO:teuthology.orchestra.run.smithi144.stderr:512 bytes copied, 0.000206898 s, 2.5 MB/s 2024-09-16T08:07:29.311 DEBUG:teuthology.orchestra.run.smithi144:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2024-09-16T08:07:29.367 DEBUG:teuthology.orchestra.run.smithi144:> stat /dev/vg_nvme/lv_3 2024-09-16T08:07:29.422 INFO:teuthology.orchestra.run.smithi144.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2024-09-16T08:07:29.422 INFO:teuthology.orchestra.run.smithi144.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-16T08:07:29.422 INFO:teuthology.orchestra.run.smithi144.stdout:Device: 5h/5d Inode: 704 Links: 1 2024-09-16T08:07:29.423 INFO:teuthology.orchestra.run.smithi144.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-16T08:07:29.423 INFO:teuthology.orchestra.run.smithi144.stdout:Context: system_u:object_r:device_t:s0 2024-09-16T08:07:29.423 INFO:teuthology.orchestra.run.smithi144.stdout:Access: 2024-09-16 08:07:25.940041022 +0000 2024-09-16T08:07:29.423 INFO:teuthology.orchestra.run.smithi144.stdout:Modify: 2024-09-16 08:04:24.828706802 +0000 2024-09-16T08:07:29.423 INFO:teuthology.orchestra.run.smithi144.stdout:Change: 2024-09-16 08:04:24.828706802 +0000 2024-09-16T08:07:29.423 INFO:teuthology.orchestra.run.smithi144.stdout: Birth: 2024-09-16 08:04:24.828706802 +0000 2024-09-16T08:07:29.423 DEBUG:teuthology.orchestra.run.smithi144:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2024-09-16T08:07:29.485 INFO:teuthology.orchestra.run.smithi144.stderr:1+0 records in 2024-09-16T08:07:29.485 INFO:teuthology.orchestra.run.smithi144.stderr:1+0 records out 2024-09-16T08:07:29.485 INFO:teuthology.orchestra.run.smithi144.stderr:512 bytes copied, 0.000205157 s, 2.5 MB/s 2024-09-16T08:07:29.486 DEBUG:teuthology.orchestra.run.smithi144:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2024-09-16T08:07:29.542 DEBUG:teuthology.orchestra.run.smithi144:> stat /dev/vg_nvme/lv_4 2024-09-16T08:07:29.597 INFO:teuthology.orchestra.run.smithi144.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2024-09-16T08:07:29.597 INFO:teuthology.orchestra.run.smithi144.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-16T08:07:29.597 INFO:teuthology.orchestra.run.smithi144.stdout:Device: 5h/5d Inode: 725 Links: 1 2024-09-16T08:07:29.598 INFO:teuthology.orchestra.run.smithi144.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-16T08:07:29.598 INFO:teuthology.orchestra.run.smithi144.stdout:Context: system_u:object_r:device_t:s0 2024-09-16T08:07:29.598 INFO:teuthology.orchestra.run.smithi144.stdout:Access: 2024-09-16 08:07:25.940041022 +0000 2024-09-16T08:07:29.598 INFO:teuthology.orchestra.run.smithi144.stdout:Modify: 2024-09-16 08:04:25.161701130 +0000 2024-09-16T08:07:29.598 INFO:teuthology.orchestra.run.smithi144.stdout:Change: 2024-09-16 08:04:25.161701130 +0000 2024-09-16T08:07:29.598 INFO:teuthology.orchestra.run.smithi144.stdout: Birth: 2024-09-16 08:04:25.161701130 +0000 2024-09-16T08:07:29.598 DEBUG:teuthology.orchestra.run.smithi144:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2024-09-16T08:07:29.660 INFO:teuthology.orchestra.run.smithi144.stderr:1+0 records in 2024-09-16T08:07:29.660 INFO:teuthology.orchestra.run.smithi144.stderr:1+0 records out 2024-09-16T08:07:29.660 INFO:teuthology.orchestra.run.smithi144.stderr:512 bytes copied, 0.000177011 s, 2.9 MB/s 2024-09-16T08:07:29.661 DEBUG:teuthology.orchestra.run.smithi144:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2024-09-16T08:07:29.717 INFO:tasks.cephadm:Deploying osd.0 on smithi028 with /dev/vg_nvme/lv_4... 2024-09-16T08:07:29.717 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5622b2fe-7402-11ef-bceb-c7b262605968 -- lvm zap /dev/vg_nvme/lv_4 2024-09-16T08:07:30.274 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:07:30 smithi028 ceph-mon[23744]: Deploying daemon prometheus.smithi028 on smithi028 2024-09-16T08:07:30.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:07:30 smithi144 ceph-mon[25668]: Deploying daemon prometheus.smithi028 on smithi028 2024-09-16T08:07:31.446 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:07:31 smithi028 ceph-mon[23744]: pgmap v6: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-16T08:07:31.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:07:31 smithi144 ceph-mon[25668]: pgmap v6: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-16T08:07:33.446 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:07:33 smithi028 ceph-mon[23744]: pgmap v7: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-16T08:07:33.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:07:33 smithi144 ceph-mon[25668]: pgmap v7: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-16T08:07:34.946 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:07:34 smithi028 ceph-mon[23744]: pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-16T08:07:34.946 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:07:34 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' 2024-09-16T08:07:35.081 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:07:34 smithi144 ceph-mon[25668]: pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-16T08:07:35.081 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:07:34 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' 2024-09-16T08:07:37.164 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:07:37 smithi028 ceph-mon[23744]: pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-16T08:07:37.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:07:37 smithi144 ceph-mon[25668]: pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-16T08:07:39.318 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:07:39 smithi028 ceph-mon[23744]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-16T08:07:39.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:07:39 smithi144 ceph-mon[25668]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-16T08:07:41.154 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:07:41 smithi028 ceph-mon[23744]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-16T08:07:41.154 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:07:41 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' 2024-09-16T08:07:41.154 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:07:41 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-16T08:07:41.154 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:07:41 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T08:07:41.155 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:07:41 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-16T08:07:41.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:07:41 smithi144 ceph-mon[25668]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-16T08:07:41.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:07:41 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' 2024-09-16T08:07:41.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:07:41 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-16T08:07:41.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:07:41 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T08:07:41.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:07:41 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-16T08:07:41.803 INFO:teuthology.orchestra.run.smithi028.stdout: 2024-09-16T08:07:41.823 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5622b2fe-7402-11ef-bceb-c7b262605968 -- ceph orch daemon add osd smithi028:vg_nvme/lv_4 2024-09-16T08:07:43.290 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:07:43 smithi028 ceph-mon[23744]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-16T08:07:43.290 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:07:43 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' 2024-09-16T08:07:43.290 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:07:43 smithi028 ceph-mon[23744]: Updating smithi144:/etc/ceph/ceph.conf 2024-09-16T08:07:43.290 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:07:43 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' 2024-09-16T08:07:43.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:07:43 smithi144 ceph-mon[25668]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-16T08:07:43.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:07:43 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' 2024-09-16T08:07:43.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:07:43 smithi144 ceph-mon[25668]: Updating smithi144:/etc/ceph/ceph.conf 2024-09-16T08:07:43.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:07:43 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' 2024-09-16T08:07:44.446 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:07:44 smithi028 ceph-mon[23744]: Updating smithi144:/etc/ceph/ceph.client.admin.keyring 2024-09-16T08:07:44.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:07:44 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' 2024-09-16T08:07:44.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:07:44 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-16T08:07:44.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:07:44 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-16T08:07:44.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:07:44 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T08:07:44.584 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:07:44 smithi144 ceph-mon[25668]: Updating smithi144:/etc/ceph/ceph.client.admin.keyring 2024-09-16T08:07:44.584 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:07:44 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' 2024-09-16T08:07:44.584 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:07:44 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-16T08:07:44.584 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:07:44 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-16T08:07:44.585 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:07:44 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T08:07:45.387 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:07:45 smithi028 ceph-mon[23744]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-16T08:07:45.387 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:07:45 smithi028 ceph-mon[23744]: from='client.14230 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi028:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T08:07:45.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:07:45 smithi144 ceph-mon[25668]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-16T08:07:45.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:07:45 smithi144 ceph-mon[25668]: from='client.14230 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi028:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T08:07:47.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:07:47 smithi028 ceph-mon[23744]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-16T08:07:47.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:07:47 smithi028 ceph-mon[23744]: from='client.? 172.21.15.28:0/4149928067' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "66f613ac-6dba-4ca1-ab30-8b95e06260c0"}]: dispatch 2024-09-16T08:07:47.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:07:47 smithi028 ceph-mon[23744]: from='client.? 172.21.15.28:0/4149928067' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "66f613ac-6dba-4ca1-ab30-8b95e06260c0"}]': finished 2024-09-16T08:07:47.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:07:47 smithi028 ceph-mon[23744]: osdmap e6: 1 total, 0 up, 1 in 2024-09-16T08:07:47.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:07:47 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-16T08:07:47.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:07:47 smithi028 ceph-mon[23744]: from='client.? 172.21.15.28:0/888109581' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-16T08:07:47.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:07:47 smithi144 ceph-mon[25668]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-16T08:07:47.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:07:47 smithi144 ceph-mon[25668]: from='client.? 172.21.15.28:0/4149928067' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "66f613ac-6dba-4ca1-ab30-8b95e06260c0"}]: dispatch 2024-09-16T08:07:47.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:07:47 smithi144 ceph-mon[25668]: from='client.? 172.21.15.28:0/4149928067' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "66f613ac-6dba-4ca1-ab30-8b95e06260c0"}]': finished 2024-09-16T08:07:47.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:07:47 smithi144 ceph-mon[25668]: osdmap e6: 1 total, 0 up, 1 in 2024-09-16T08:07:47.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:07:47 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-16T08:07:47.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:07:47 smithi144 ceph-mon[25668]: from='client.? 172.21.15.28:0/888109581' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-16T08:07:49.285 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:07:49 smithi028 ceph-mon[23744]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-16T08:07:49.285 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:07:49 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' 2024-09-16T08:07:49.285 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:07:49 smithi028 ceph-mon[23744]: Updating smithi028:/etc/ceph/ceph.conf 2024-09-16T08:07:49.285 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:07:49 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' 2024-09-16T08:07:49.285 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:07:49 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' 2024-09-16T08:07:49.285 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:07:49 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-16T08:07:49.285 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:07:49 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-16T08:07:49.285 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:07:49 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T08:07:49.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:07:49 smithi144 ceph-mon[25668]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-16T08:07:49.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:07:49 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' 2024-09-16T08:07:49.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:07:49 smithi144 ceph-mon[25668]: Updating smithi028:/etc/ceph/ceph.conf 2024-09-16T08:07:49.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:07:49 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' 2024-09-16T08:07:49.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:07:49 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' 2024-09-16T08:07:49.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:07:49 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-16T08:07:49.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:07:49 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-16T08:07:49.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:07:49 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T08:07:50.158 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:07:50 smithi028 ceph-mon[23744]: Reconfiguring mon.smithi028 (unknown last config time)... 2024-09-16T08:07:50.158 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:07:50 smithi028 ceph-mon[23744]: Reconfiguring daemon mon.smithi028 on smithi028 2024-09-16T08:07:50.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:07:50 smithi144 ceph-mon[25668]: Reconfiguring mon.smithi028 (unknown last config time)... 2024-09-16T08:07:50.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:07:50 smithi144 ceph-mon[25668]: Reconfiguring daemon mon.smithi028 on smithi028 2024-09-16T08:07:51.300 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:07:51 smithi028 ceph-mon[23744]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-16T08:07:51.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:07:51 smithi144 ceph-mon[25668]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-16T08:07:53.311 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:07:53 smithi028 ceph-mon[23744]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-16T08:07:53.311 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:07:53 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' 2024-09-16T08:07:53.311 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:07:53 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi028.azvyzy", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-16T08:07:53.311 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:07:53 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-16T08:07:53.311 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:07:53 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T08:07:53.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:07:53 smithi144 ceph-mon[25668]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-16T08:07:53.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:07:53 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' 2024-09-16T08:07:53.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:07:53 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi028.azvyzy", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-16T08:07:53.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:07:53 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-16T08:07:53.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:07:53 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T08:07:54.100 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:07:54 smithi028 ceph-mon[23744]: Reconfiguring mgr.smithi028.azvyzy (unknown last config time)... 2024-09-16T08:07:54.100 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:07:54 smithi028 ceph-mon[23744]: Reconfiguring daemon mgr.smithi028.azvyzy on smithi028 2024-09-16T08:07:54.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:07:54 smithi144 ceph-mon[25668]: Reconfiguring mgr.smithi028.azvyzy (unknown last config time)... 2024-09-16T08:07:54.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:07:54 smithi144 ceph-mon[25668]: Reconfiguring daemon mgr.smithi028.azvyzy on smithi028 2024-09-16T08:07:55.232 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:07:55 smithi028 ceph-mon[23744]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-16T08:07:55.232 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:07:55 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' 2024-09-16T08:07:55.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:07:55 smithi144 ceph-mon[25668]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-16T08:07:55.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:07:55 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' 2024-09-16T08:07:56.107 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:07:56 smithi028 ceph-mon[23744]: Reconfiguring alertmanager.smithi028 (dependencies changed)... 2024-09-16T08:07:56.108 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:07:56 smithi028 ceph-mon[23744]: Reconfiguring daemon alertmanager.smithi028 on smithi028 2024-09-16T08:07:56.108 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:07:56 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-09-16T08:07:56.108 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:07:56 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T08:07:56.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:07:56 smithi144 ceph-mon[25668]: Reconfiguring alertmanager.smithi028 (dependencies changed)... 2024-09-16T08:07:56.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:07:56 smithi144 ceph-mon[25668]: Reconfiguring daemon alertmanager.smithi028 on smithi028 2024-09-16T08:07:56.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:07:56 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-09-16T08:07:56.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:07:56 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T08:07:57.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:07:57 smithi144 ceph-mon[25668]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-16T08:07:57.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:07:57 smithi144 ceph-mon[25668]: Deploying daemon osd.0 on smithi028 2024-09-16T08:07:57.353 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:07:57 smithi028 ceph-mon[23744]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-16T08:07:57.353 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:07:57 smithi028 ceph-mon[23744]: Deploying daemon osd.0 on smithi028 2024-09-16T08:07:59.118 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:07:59 smithi028 ceph-mon[23744]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-16T08:07:59.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:07:59 smithi144 ceph-mon[25668]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-16T08:08:01.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:08:01 smithi144 ceph-mon[25668]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-16T08:08:01.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:08:01 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' 2024-09-16T08:08:01.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:08:01 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi028", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-09-16T08:08:01.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:08:01 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T08:08:01.372 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:08:01 smithi028 ceph-mon[23744]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-16T08:08:01.372 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:08:01 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' 2024-09-16T08:08:01.372 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:08:01 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi028", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-09-16T08:08:01.372 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:08:01 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T08:08:02.196 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:08:02 smithi028 ceph-mon[23744]: Reconfiguring crash.smithi028 (monmap changed)... 2024-09-16T08:08:02.196 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:08:02 smithi028 ceph-mon[23744]: Reconfiguring daemon crash.smithi028 on smithi028 2024-09-16T08:08:02.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:08:02 smithi144 ceph-mon[25668]: Reconfiguring crash.smithi028 (monmap changed)... 2024-09-16T08:08:02.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:08:02 smithi144 ceph-mon[25668]: Reconfiguring daemon crash.smithi028 on smithi028 2024-09-16T08:08:03.197 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:08:03 smithi028 ceph-mon[23744]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-16T08:08:03.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:08:03 smithi144 ceph-mon[25668]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-16T08:08:04.196 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:08:04 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi028.azvyzy/mirror_snapshot_schedule"}]: dispatch 2024-09-16T08:08:04.196 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:08:04 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi028.azvyzy/trash_purge_schedule"}]: dispatch 2024-09-16T08:08:04.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:08:04 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi028.azvyzy/mirror_snapshot_schedule"}]: dispatch 2024-09-16T08:08:04.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:08:04 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi028.azvyzy/trash_purge_schedule"}]: dispatch 2024-09-16T08:08:05.196 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:08:05 smithi028 ceph-mon[23744]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-16T08:08:05.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:08:05 smithi144 ceph-mon[25668]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-16T08:08:06.111 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:08:06 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' 2024-09-16T08:08:06.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:08:06 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' 2024-09-16T08:08:07.259 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:08:07 smithi028 ceph-mon[23744]: pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-16T08:08:07.260 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:08:07 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' 2024-09-16T08:08:07.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:08:07 smithi144 ceph-mon[25668]: pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-16T08:08:07.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:08:07 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' 2024-09-16T08:08:08.196 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:08:08 smithi028 ceph-mon[23744]: Reconfiguring grafana.smithi028 (dependencies changed)... 2024-09-16T08:08:08.196 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:08:08 smithi028 ceph-mon[23744]: Reconfiguring daemon grafana.smithi028 on smithi028 2024-09-16T08:08:08.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:08:08 smithi144 ceph-mon[25668]: Reconfiguring grafana.smithi028 (dependencies changed)... 2024-09-16T08:08:08.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:08:08 smithi144 ceph-mon[25668]: Reconfiguring daemon grafana.smithi028 on smithi028 2024-09-16T08:08:08.604 INFO:teuthology.orchestra.run.smithi028.stdout:Created osd(s) 0 on host 'smithi028' 2024-09-16T08:08:09.197 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:08:09 smithi028 ceph-mon[23744]: pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-16T08:08:09.197 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:08:09 smithi028 ceph-mon[23744]: from='osd.0 [v2:172.21.15.28:6802/230701446,v1:172.21.15.28:6803/230701446]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-09-16T08:08:09.197 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:08:09 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' 2024-09-16T08:08:09.436 DEBUG:teuthology.orchestra.run.smithi028:osd.0> sudo journalctl -f -n 0 -u ceph-5622b2fe-7402-11ef-bceb-c7b262605968@osd.0.service 2024-09-16T08:08:09.439 INFO:tasks.cephadm:Deploying osd.1 on smithi028 with /dev/vg_nvme/lv_3... 2024-09-16T08:08:09.439 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5622b2fe-7402-11ef-bceb-c7b262605968 -- lvm zap /dev/vg_nvme/lv_3 2024-09-16T08:08:09.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:08:09 smithi144 ceph-mon[25668]: pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-16T08:08:09.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:08:09 smithi144 ceph-mon[25668]: from='osd.0 [v2:172.21.15.28:6802/230701446,v1:172.21.15.28:6803/230701446]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-09-16T08:08:09.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:08:09 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' 2024-09-16T08:08:10.211 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:08:10 smithi028 ceph-mon[23744]: from='osd.0 [v2:172.21.15.28:6802/230701446,v1:172.21.15.28:6803/230701446]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-09-16T08:08:10.212 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:08:10 smithi028 ceph-mon[23744]: osdmap e7: 1 total, 0 up, 1 in 2024-09-16T08:08:10.212 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:08:10 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-16T08:08:10.212 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:08:10 smithi028 ceph-mon[23744]: from='osd.0 [v2:172.21.15.28:6802/230701446,v1:172.21.15.28:6803/230701446]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi028", "root=default"]}]: dispatch 2024-09-16T08:08:10.212 INFO:journalctl@ceph.osd.0.smithi028.stdout:Sep 16 08:08:10 smithi028 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-0[34239]: 2024-09-16T08:08:10.103+0000 7f65e701f700 -1 osd.0 0 waiting for initial osdmap 2024-09-16T08:08:10.212 INFO:journalctl@ceph.osd.0.smithi028.stdout:Sep 16 08:08:10 smithi028 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-0[34239]: 2024-09-16T08:08:10.106+0000 7f65df9b2700 -1 osd.0 8 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-16T08:08:10.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:08:10 smithi144 ceph-mon[25668]: from='osd.0 [v2:172.21.15.28:6802/230701446,v1:172.21.15.28:6803/230701446]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-09-16T08:08:10.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:08:10 smithi144 ceph-mon[25668]: osdmap e7: 1 total, 0 up, 1 in 2024-09-16T08:08:10.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:08:10 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-16T08:08:10.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:08:10 smithi144 ceph-mon[25668]: from='osd.0 [v2:172.21.15.28:6802/230701446,v1:172.21.15.28:6803/230701446]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi028", "root=default"]}]: dispatch 2024-09-16T08:08:11.128 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:08:11 smithi028 ceph-mon[23744]: pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-16T08:08:11.128 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:08:11 smithi028 ceph-mon[23744]: from='osd.0 [v2:172.21.15.28:6802/230701446,v1:172.21.15.28:6803/230701446]' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi028", "root=default"]}]': finished 2024-09-16T08:08:11.128 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:08:11 smithi028 ceph-mon[23744]: osdmap e8: 1 total, 0 up, 1 in 2024-09-16T08:08:11.128 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:08:11 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-16T08:08:11.128 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:08:11 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-16T08:08:11.128 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:08:11 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-16T08:08:11.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:08:11 smithi144 ceph-mon[25668]: pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-16T08:08:11.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:08:11 smithi144 ceph-mon[25668]: from='osd.0 [v2:172.21.15.28:6802/230701446,v1:172.21.15.28:6803/230701446]' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi028", "root=default"]}]': finished 2024-09-16T08:08:11.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:08:11 smithi144 ceph-mon[25668]: osdmap e8: 1 total, 0 up, 1 in 2024-09-16T08:08:11.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:08:11 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-16T08:08:11.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:08:11 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-16T08:08:11.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:08:11 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-16T08:08:12.165 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:08:12 smithi028 ceph-mon[23744]: purged_snaps scrub starts 2024-09-16T08:08:12.165 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:08:12 smithi028 ceph-mon[23744]: purged_snaps scrub ok 2024-09-16T08:08:12.165 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:08:12 smithi028 ceph-mon[23744]: osd.0 [v2:172.21.15.28:6802/230701446,v1:172.21.15.28:6803/230701446] boot 2024-09-16T08:08:12.165 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:08:12 smithi028 ceph-mon[23744]: osdmap e9: 1 total, 1 up, 1 in 2024-09-16T08:08:12.166 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:08:12 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-16T08:08:12.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:08:12 smithi144 ceph-mon[25668]: purged_snaps scrub starts 2024-09-16T08:08:12.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:08:12 smithi144 ceph-mon[25668]: purged_snaps scrub ok 2024-09-16T08:08:12.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:08:12 smithi144 ceph-mon[25668]: osd.0 [v2:172.21.15.28:6802/230701446,v1:172.21.15.28:6803/230701446] boot 2024-09-16T08:08:12.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:08:12 smithi144 ceph-mon[25668]: osdmap e9: 1 total, 1 up, 1 in 2024-09-16T08:08:12.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:08:12 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-16T08:08:13.278 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:08:13 smithi028 ceph-mon[23744]: pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-16T08:08:13.278 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:08:13 smithi028 ceph-mon[23744]: osdmap e10: 1 total, 1 up, 1 in 2024-09-16T08:08:13.449 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:08:13 smithi144 ceph-mon[25668]: pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-16T08:08:13.449 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:08:13 smithi144 ceph-mon[25668]: osdmap e10: 1 total, 1 up, 1 in 2024-09-16T08:08:14.947 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:08:14 smithi028 ceph-mon[23744]: pgmap v33: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-16T08:08:14.947 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:08:14 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' 2024-09-16T08:08:14.947 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:08:14 smithi028 ceph-mon[23744]: Reconfiguring crash.smithi144 (monmap changed)... 2024-09-16T08:08:14.947 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:08:14 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi144", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-09-16T08:08:14.947 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:08:14 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T08:08:14.947 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:08:14 smithi028 ceph-mon[23744]: Reconfiguring daemon crash.smithi144 on smithi144 2024-09-16T08:08:14.966 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:08:14 smithi144 ceph-mon[25668]: pgmap v33: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-16T08:08:14.966 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:08:14 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' 2024-09-16T08:08:14.966 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:08:14 smithi144 ceph-mon[25668]: Reconfiguring crash.smithi144 (monmap changed)... 2024-09-16T08:08:14.966 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:08:14 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi144", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-09-16T08:08:14.966 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:08:14 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T08:08:14.966 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:08:14 smithi144 ceph-mon[25668]: Reconfiguring daemon crash.smithi144 on smithi144 2024-09-16T08:08:15.037 INFO:teuthology.orchestra.run.smithi028.stdout: 2024-09-16T08:08:15.059 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5622b2fe-7402-11ef-bceb-c7b262605968 -- ceph orch daemon add osd smithi028:vg_nvme/lv_3 2024-09-16T08:08:16.186 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:08:16 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' 2024-09-16T08:08:16.187 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:08:16 smithi028 ceph-mon[23744]: Reconfiguring mgr.smithi144.utzszj (monmap changed)... 2024-09-16T08:08:16.187 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:08:16 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi144.utzszj", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-16T08:08:16.187 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:08:16 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-16T08:08:16.187 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:08:16 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T08:08:16.187 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:08:16 smithi028 ceph-mon[23744]: Reconfiguring daemon mgr.smithi144.utzszj on smithi144 2024-09-16T08:08:16.272 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:08:16 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' 2024-09-16T08:08:16.272 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:08:16 smithi144 ceph-mon[25668]: Reconfiguring mgr.smithi144.utzszj (monmap changed)... 2024-09-16T08:08:16.272 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:08:16 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi144.utzszj", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-16T08:08:16.272 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:08:16 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-16T08:08:16.272 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:08:16 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T08:08:16.273 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:08:16 smithi144 ceph-mon[25668]: Reconfiguring daemon mgr.smithi144.utzszj on smithi144 2024-09-16T08:08:17.196 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:08:17 smithi028 ceph-mon[23744]: pgmap v34: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-16T08:08:17.197 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:08:17 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' 2024-09-16T08:08:17.197 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:08:17 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-16T08:08:17.197 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:08:17 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-16T08:08:17.197 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:08:17 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T08:08:17.197 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:08:17 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-16T08:08:17.197 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:08:17 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-16T08:08:17.197 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:08:17 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T08:08:17.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:08:17 smithi144 ceph-mon[25668]: pgmap v34: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-16T08:08:17.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:08:17 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' 2024-09-16T08:08:17.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:08:17 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-16T08:08:17.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:08:17 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-16T08:08:17.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:08:17 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T08:08:17.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:08:17 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-16T08:08:17.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:08:17 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-16T08:08:17.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:08:17 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T08:08:18.197 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:08:18 smithi028 ceph-mon[23744]: Reconfiguring mon.smithi144 (monmap changed)... 2024-09-16T08:08:18.197 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:08:18 smithi028 ceph-mon[23744]: Reconfiguring daemon mon.smithi144 on smithi144 2024-09-16T08:08:18.197 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:08:18 smithi028 ceph-mon[23744]: from='client.14248 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi028:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T08:08:18.197 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:08:18 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' 2024-09-16T08:08:18.197 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:08:18 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-16T08:08:18.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:08:18 smithi144 ceph-mon[25668]: Reconfiguring mon.smithi144 (monmap changed)... 2024-09-16T08:08:18.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:08:18 smithi144 ceph-mon[25668]: Reconfiguring daemon mon.smithi144 on smithi144 2024-09-16T08:08:18.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:08:18 smithi144 ceph-mon[25668]: from='client.14248 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi028:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T08:08:18.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:08:18 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' 2024-09-16T08:08:18.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:08:18 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-16T08:08:19.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:08:19 smithi144 ceph-mon[25668]: pgmap v35: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-16T08:08:19.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:08:19 smithi144 ceph-mon[25668]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-16T08:08:19.446 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:08:19 smithi028 ceph-mon[23744]: pgmap v35: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-16T08:08:19.446 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:08:19 smithi028 ceph-mon[23744]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-16T08:08:20.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:08:20 smithi144 ceph-mon[25668]: from='client.? 172.21.15.28:0/145378610' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "524eec0a-00eb-403b-a964-ae56bd9eb6f5"}]: dispatch 2024-09-16T08:08:20.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:08:20 smithi144 ceph-mon[25668]: from='client.? 172.21.15.28:0/145378610' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "524eec0a-00eb-403b-a964-ae56bd9eb6f5"}]': finished 2024-09-16T08:08:20.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:08:20 smithi144 ceph-mon[25668]: osdmap e11: 2 total, 1 up, 2 in 2024-09-16T08:08:20.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:08:20 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-16T08:08:20.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:08:20 smithi144 ceph-mon[25668]: from='client.? 172.21.15.28:0/1370518560' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-16T08:08:20.446 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:08:20 smithi028 ceph-mon[23744]: from='client.? 172.21.15.28:0/145378610' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "524eec0a-00eb-403b-a964-ae56bd9eb6f5"}]: dispatch 2024-09-16T08:08:20.446 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:08:20 smithi028 ceph-mon[23744]: from='client.? 172.21.15.28:0/145378610' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "524eec0a-00eb-403b-a964-ae56bd9eb6f5"}]': finished 2024-09-16T08:08:20.446 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:08:20 smithi028 ceph-mon[23744]: osdmap e11: 2 total, 1 up, 2 in 2024-09-16T08:08:20.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:08:20 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-16T08:08:20.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:08:20 smithi028 ceph-mon[23744]: from='client.? 172.21.15.28:0/1370518560' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-16T08:08:21.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:08:21 smithi144 ceph-mon[25668]: pgmap v37: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-16T08:08:21.446 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:08:21 smithi028 ceph-mon[23744]: pgmap v37: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-16T08:08:23.297 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:08:23 smithi028 ceph-mon[23744]: pgmap v38: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-16T08:08:23.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:08:23 smithi144 ceph-mon[25668]: pgmap v38: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-16T08:08:25.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:08:25 smithi144 ceph-mon[25668]: pgmap v39: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-16T08:08:25.342 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:08:25 smithi028 ceph-mon[23744]: pgmap v39: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-16T08:08:26.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:08:26 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-09-16T08:08:26.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:08:26 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T08:08:26.446 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:08:26 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-09-16T08:08:26.446 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:08:26 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T08:08:27.281 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:08:27 smithi028 ceph-mon[23744]: Deploying daemon osd.1 on smithi028 2024-09-16T08:08:27.281 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:08:27 smithi028 ceph-mon[23744]: pgmap v40: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-16T08:08:27.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:08:27 smithi144 ceph-mon[25668]: Deploying daemon osd.1 on smithi028 2024-09-16T08:08:27.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:08:27 smithi144 ceph-mon[25668]: pgmap v40: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-16T08:08:29.196 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:08:29 smithi028 ceph-mon[23744]: pgmap v41: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-16T08:08:29.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:08:29 smithi144 ceph-mon[25668]: pgmap v41: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-16T08:08:31.322 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:08:31 smithi028 ceph-mon[23744]: pgmap v42: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-16T08:08:31.323 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:08:31 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' 2024-09-16T08:08:31.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:08:31 smithi144 ceph-mon[25668]: pgmap v42: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-16T08:08:31.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:08:31 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' 2024-09-16T08:08:32.976 INFO:teuthology.orchestra.run.smithi028.stdout:Created osd(s) 1 on host 'smithi028' 2024-09-16T08:08:33.197 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:08:33 smithi028 ceph-mon[23744]: pgmap v43: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-16T08:08:33.197 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:08:33 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' 2024-09-16T08:08:33.197 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:08:33 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.28:9093"}]: dispatch 2024-09-16T08:08:33.197 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:08:33 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' 2024-09-16T08:08:33.197 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:08:33 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-16T08:08:33.197 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:08:33 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.28:3000"}]: dispatch 2024-09-16T08:08:33.197 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:08:33 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' 2024-09-16T08:08:33.197 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:08:33 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-16T08:08:33.197 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:08:33 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.28:9095"}]: dispatch 2024-09-16T08:08:33.197 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:08:33 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' 2024-09-16T08:08:33.198 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:08:33 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-16T08:08:33.198 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:08:33 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T08:08:33.198 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:08:33 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-16T08:08:33.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:08:33 smithi144 ceph-mon[25668]: pgmap v43: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-16T08:08:33.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:08:33 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' 2024-09-16T08:08:33.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:08:33 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.28:9093"}]: dispatch 2024-09-16T08:08:33.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:08:33 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' 2024-09-16T08:08:33.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:08:33 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-16T08:08:33.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:08:33 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.28:3000"}]: dispatch 2024-09-16T08:08:33.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:08:33 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' 2024-09-16T08:08:33.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:08:33 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-16T08:08:33.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:08:33 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.28:9095"}]: dispatch 2024-09-16T08:08:33.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:08:33 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' 2024-09-16T08:08:33.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:08:33 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-16T08:08:33.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:08:33 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T08:08:33.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:08:33 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-16T08:08:33.598 DEBUG:teuthology.orchestra.run.smithi028:osd.1> sudo journalctl -f -n 0 -u ceph-5622b2fe-7402-11ef-bceb-c7b262605968@osd.1.service 2024-09-16T08:08:33.601 INFO:tasks.cephadm:Deploying osd.2 on smithi028 with /dev/vg_nvme/lv_2... 2024-09-16T08:08:33.601 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5622b2fe-7402-11ef-bceb-c7b262605968 -- lvm zap /dev/vg_nvme/lv_2 2024-09-16T08:08:34.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:08:34 smithi144 ceph-mon[25668]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.28:9093"}]: dispatch 2024-09-16T08:08:34.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:08:34 smithi144 ceph-mon[25668]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-16T08:08:34.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:08:34 smithi144 ceph-mon[25668]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.28:3000"}]: dispatch 2024-09-16T08:08:34.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:08:34 smithi144 ceph-mon[25668]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-16T08:08:34.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:08:34 smithi144 ceph-mon[25668]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.28:9095"}]: dispatch 2024-09-16T08:08:34.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:08:34 smithi144 ceph-mon[25668]: from='osd.1 [v2:172.21.15.28:6810/3124900380,v1:172.21.15.28:6811/3124900380]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-09-16T08:08:34.446 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:08:34 smithi028 ceph-mon[23744]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.28:9093"}]: dispatch 2024-09-16T08:08:34.446 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:08:34 smithi028 ceph-mon[23744]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-16T08:08:34.446 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:08:34 smithi028 ceph-mon[23744]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.28:3000"}]: dispatch 2024-09-16T08:08:34.446 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:08:34 smithi028 ceph-mon[23744]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-16T08:08:34.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:08:34 smithi028 ceph-mon[23744]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.28:9095"}]: dispatch 2024-09-16T08:08:34.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:08:34 smithi028 ceph-mon[23744]: from='osd.1 [v2:172.21.15.28:6810/3124900380,v1:172.21.15.28:6811/3124900380]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-09-16T08:08:35.115 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:08:35 smithi028 ceph-mon[23744]: pgmap v44: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-16T08:08:35.115 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:08:35 smithi028 ceph-mon[23744]: from='osd.1 [v2:172.21.15.28:6810/3124900380,v1:172.21.15.28:6811/3124900380]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-09-16T08:08:35.115 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:08:35 smithi028 ceph-mon[23744]: osdmap e12: 2 total, 1 up, 2 in 2024-09-16T08:08:35.115 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:08:35 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-16T08:08:35.115 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:08:35 smithi028 ceph-mon[23744]: from='osd.1 [v2:172.21.15.28:6810/3124900380,v1:172.21.15.28:6811/3124900380]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi028", "root=default"]}]: dispatch 2024-09-16T08:08:35.116 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:08:35 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' 2024-09-16T08:08:35.116 INFO:journalctl@ceph.osd.1.smithi028.stdout:Sep 16 08:08:35 smithi028 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-1[37067]: 2024-09-16T08:08:35.069+0000 7f586ada1700 -1 osd.1 0 waiting for initial osdmap 2024-09-16T08:08:35.116 INFO:journalctl@ceph.osd.1.smithi028.stdout:Sep 16 08:08:35 smithi028 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-1[37067]: 2024-09-16T08:08:35.072+0000 7f5865f39700 -1 osd.1 13 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-16T08:08:35.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:08:35 smithi144 ceph-mon[25668]: pgmap v44: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-16T08:08:35.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:08:35 smithi144 ceph-mon[25668]: from='osd.1 [v2:172.21.15.28:6810/3124900380,v1:172.21.15.28:6811/3124900380]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-09-16T08:08:35.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:08:35 smithi144 ceph-mon[25668]: osdmap e12: 2 total, 1 up, 2 in 2024-09-16T08:08:35.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:08:35 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-16T08:08:35.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:08:35 smithi144 ceph-mon[25668]: from='osd.1 [v2:172.21.15.28:6810/3124900380,v1:172.21.15.28:6811/3124900380]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi028", "root=default"]}]: dispatch 2024-09-16T08:08:35.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:08:35 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' 2024-09-16T08:08:36.257 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:08:36 smithi028 ceph-mon[23744]: from='osd.1 [v2:172.21.15.28:6810/3124900380,v1:172.21.15.28:6811/3124900380]' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi028", "root=default"]}]': finished 2024-09-16T08:08:36.257 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:08:36 smithi028 ceph-mon[23744]: osdmap e13: 2 total, 1 up, 2 in 2024-09-16T08:08:36.257 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:08:36 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-16T08:08:36.257 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:08:36 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-16T08:08:36.257 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:08:36 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-16T08:08:36.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:08:36 smithi144 ceph-mon[25668]: from='osd.1 [v2:172.21.15.28:6810/3124900380,v1:172.21.15.28:6811/3124900380]' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi028", "root=default"]}]': finished 2024-09-16T08:08:36.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:08:36 smithi144 ceph-mon[25668]: osdmap e13: 2 total, 1 up, 2 in 2024-09-16T08:08:36.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:08:36 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-16T08:08:36.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:08:36 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-16T08:08:36.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:08:36 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-16T08:08:37.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:08:37 smithi144 ceph-mon[25668]: purged_snaps scrub starts 2024-09-16T08:08:37.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:08:37 smithi144 ceph-mon[25668]: purged_snaps scrub ok 2024-09-16T08:08:37.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:08:37 smithi144 ceph-mon[25668]: pgmap v47: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-16T08:08:37.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:08:37 smithi144 ceph-mon[25668]: osd.1 [v2:172.21.15.28:6810/3124900380,v1:172.21.15.28:6811/3124900380] boot 2024-09-16T08:08:37.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:08:37 smithi144 ceph-mon[25668]: osdmap e14: 2 total, 2 up, 2 in 2024-09-16T08:08:37.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:08:37 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-16T08:08:37.380 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:08:37 smithi028 ceph-mon[23744]: purged_snaps scrub starts 2024-09-16T08:08:37.380 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:08:37 smithi028 ceph-mon[23744]: purged_snaps scrub ok 2024-09-16T08:08:37.381 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:08:37 smithi028 ceph-mon[23744]: pgmap v47: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-16T08:08:37.381 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:08:37 smithi028 ceph-mon[23744]: osd.1 [v2:172.21.15.28:6810/3124900380,v1:172.21.15.28:6811/3124900380] boot 2024-09-16T08:08:37.381 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:08:37 smithi028 ceph-mon[23744]: osdmap e14: 2 total, 2 up, 2 in 2024-09-16T08:08:37.381 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:08:37 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-16T08:08:37.665 INFO:teuthology.orchestra.run.smithi028.stdout: 2024-09-16T08:08:37.687 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5622b2fe-7402-11ef-bceb-c7b262605968 -- ceph orch daemon add osd smithi028:vg_nvme/lv_2 2024-09-16T08:08:38.310 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:08:38 smithi028 ceph-mon[23744]: osdmap e15: 2 total, 2 up, 2 in 2024-09-16T08:08:38.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:08:38 smithi144 ceph-mon[25668]: osdmap e15: 2 total, 2 up, 2 in 2024-09-16T08:08:39.314 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:08:39 smithi028 ceph-mon[23744]: pgmap v50: 0 pgs: ; 0 B data, 9.7 MiB used, 179 GiB / 179 GiB avail 2024-09-16T08:08:39.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:08:39 smithi144 ceph-mon[25668]: pgmap v50: 0 pgs: ; 0 B data, 9.7 MiB used, 179 GiB / 179 GiB avail 2024-09-16T08:08:41.303 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:08:41 smithi028 ceph-mon[23744]: pgmap v51: 0 pgs: ; 0 B data, 9.7 MiB used, 179 GiB / 179 GiB avail 2024-09-16T08:08:41.303 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:08:41 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' 2024-09-16T08:08:41.303 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:08:41 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-16T08:08:41.303 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:08:41 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-16T08:08:41.303 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:08:41 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T08:08:41.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:08:41 smithi144 ceph-mon[25668]: pgmap v51: 0 pgs: ; 0 B data, 9.7 MiB used, 179 GiB / 179 GiB avail 2024-09-16T08:08:41.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:08:41 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' 2024-09-16T08:08:41.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:08:41 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-16T08:08:41.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:08:41 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-16T08:08:41.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:08:41 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T08:08:42.190 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:08:42 smithi028 ceph-mon[23744]: from='client.14266 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi028:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T08:08:42.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:08:42 smithi144 ceph-mon[25668]: from='client.14266 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi028:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T08:08:43.246 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:08:43 smithi028 ceph-mon[23744]: pgmap v52: 0 pgs: ; 0 B data, 9.7 MiB used, 179 GiB / 179 GiB avail 2024-09-16T08:08:43.320 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:08:43 smithi144 ceph-mon[25668]: pgmap v52: 0 pgs: ; 0 B data, 9.7 MiB used, 179 GiB / 179 GiB avail 2024-09-16T08:08:44.561 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:08:44 smithi144 ceph-mon[25668]: Detected new or changed devices on smithi028 2024-09-16T08:08:44.561 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:08:44 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' 2024-09-16T08:08:44.561 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:08:44 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-16T08:08:44.561 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:08:44 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-16T08:08:44.561 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:08:44 smithi144 ceph-mon[25668]: Adjusting osd_memory_target on smithi028 to 6434M 2024-09-16T08:08:44.561 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:08:44 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' 2024-09-16T08:08:44.561 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:08:44 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' 2024-09-16T08:08:44.561 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:08:44 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-16T08:08:44.562 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:08:44 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T08:08:44.562 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:08:44 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-16T08:08:44.562 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:08:44 smithi144 ceph-mon[25668]: pgmap v53: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-16T08:08:44.592 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:08:44 smithi028 ceph-mon[23744]: Detected new or changed devices on smithi028 2024-09-16T08:08:44.593 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:08:44 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' 2024-09-16T08:08:44.593 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:08:44 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-16T08:08:44.593 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:08:44 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-16T08:08:44.593 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:08:44 smithi028 ceph-mon[23744]: Adjusting osd_memory_target on smithi028 to 6434M 2024-09-16T08:08:44.593 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:08:44 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' 2024-09-16T08:08:44.593 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:08:44 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' 2024-09-16T08:08:44.593 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:08:44 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-16T08:08:44.593 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:08:44 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T08:08:44.593 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:08:44 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-16T08:08:44.593 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:08:44 smithi028 ceph-mon[23744]: pgmap v53: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-16T08:08:45.448 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:08:45 smithi028 ceph-mon[23744]: from='client.? 172.21.15.28:0/606247739' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "cabe96b6-9d98-487a-8a56-6ef052c609f1"}]: dispatch 2024-09-16T08:08:45.448 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:08:45 smithi028 ceph-mon[23744]: from='client.? 172.21.15.28:0/606247739' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "cabe96b6-9d98-487a-8a56-6ef052c609f1"}]': finished 2024-09-16T08:08:45.448 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:08:45 smithi028 ceph-mon[23744]: osdmap e16: 3 total, 2 up, 3 in 2024-09-16T08:08:45.448 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:08:45 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-16T08:08:45.448 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:08:45 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' 2024-09-16T08:08:45.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:08:45 smithi144 ceph-mon[25668]: from='client.? 172.21.15.28:0/606247739' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "cabe96b6-9d98-487a-8a56-6ef052c609f1"}]: dispatch 2024-09-16T08:08:45.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:08:45 smithi144 ceph-mon[25668]: from='client.? 172.21.15.28:0/606247739' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "cabe96b6-9d98-487a-8a56-6ef052c609f1"}]': finished 2024-09-16T08:08:45.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:08:45 smithi144 ceph-mon[25668]: osdmap e16: 3 total, 2 up, 3 in 2024-09-16T08:08:45.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:08:45 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-16T08:08:45.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:08:45 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' 2024-09-16T08:08:46.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:08:46 smithi144 ceph-mon[25668]: from='client.? 172.21.15.28:0/2664814475' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-16T08:08:46.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:08:46 smithi144 ceph-mon[25668]: pgmap v55: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-16T08:08:46.582 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:08:46 smithi028 ceph-mon[23744]: from='client.? 172.21.15.28:0/2664814475' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-16T08:08:46.583 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:08:46 smithi028 ceph-mon[23744]: pgmap v55: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-16T08:08:48.945 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:08:48 smithi028 ceph-mon[23744]: pgmap v56: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-16T08:08:48.945 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:08:48 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' 2024-09-16T08:08:49.081 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:08:48 smithi144 ceph-mon[25668]: pgmap v56: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-16T08:08:49.081 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:08:48 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' 2024-09-16T08:08:51.196 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:08:51 smithi028 ceph-mon[23744]: pgmap v57: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-16T08:08:51.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:08:51 smithi144 ceph-mon[25668]: pgmap v57: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-16T08:08:52.196 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:08:52 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-09-16T08:08:52.197 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:08:52 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T08:08:52.197 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:08:52 smithi028 ceph-mon[23744]: Deploying daemon osd.2 on smithi028 2024-09-16T08:08:52.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:08:52 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-09-16T08:08:52.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:08:52 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T08:08:52.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:08:52 smithi144 ceph-mon[25668]: Deploying daemon osd.2 on smithi028 2024-09-16T08:08:53.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:08:53 smithi144 ceph-mon[25668]: pgmap v58: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-16T08:08:53.360 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:08:53 smithi028 ceph-mon[23744]: pgmap v58: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-16T08:08:54.446 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:08:54 smithi028 ceph-mon[23744]: Detected new or changed devices on smithi028 2024-09-16T08:08:54.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:08:54 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' 2024-09-16T08:08:54.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:08:54 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' 2024-09-16T08:08:54.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:08:54 smithi028 ceph-mon[23744]: pgmap v59: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-16T08:08:54.831 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:08:54 smithi144 ceph-mon[25668]: Detected new or changed devices on smithi028 2024-09-16T08:08:54.831 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:08:54 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' 2024-09-16T08:08:54.831 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:08:54 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' 2024-09-16T08:08:54.831 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:08:54 smithi144 ceph-mon[25668]: pgmap v59: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-16T08:08:57.197 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:08:57 smithi028 ceph-mon[23744]: pgmap v60: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-16T08:08:57.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:08:57 smithi144 ceph-mon[25668]: pgmap v60: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-16T08:08:59.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:08:59 smithi144 ceph-mon[25668]: pgmap v61: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-16T08:08:59.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:08:59 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' 2024-09-16T08:08:59.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:08:59 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-16T08:08:59.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:08:59 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T08:08:59.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:08:59 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-16T08:08:59.415 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:08:59 smithi028 ceph-mon[23744]: pgmap v61: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-16T08:08:59.415 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:08:59 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' 2024-09-16T08:08:59.415 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:08:59 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-16T08:08:59.415 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:08:59 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T08:08:59.415 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:08:59 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-16T08:09:01.081 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:09:00 smithi144 ceph-mon[25668]: pgmap v62: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-16T08:09:01.081 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:09:00 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' 2024-09-16T08:09:01.134 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:09:00 smithi028 ceph-mon[23744]: pgmap v62: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-16T08:09:01.134 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:09:00 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' 2024-09-16T08:09:01.173 INFO:teuthology.orchestra.run.smithi028.stdout:Created osd(s) 2 on host 'smithi028' 2024-09-16T08:09:02.440 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:09:02 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' 2024-09-16T08:09:02.441 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:09:02 smithi028 ceph-mon[23744]: from='osd.2 [v2:172.21.15.28:6818/3138093499,v1:172.21.15.28:6819/3138093499]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-09-16T08:09:02.467 DEBUG:teuthology.orchestra.run.smithi028:osd.2> sudo journalctl -f -n 0 -u ceph-5622b2fe-7402-11ef-bceb-c7b262605968@osd.2.service 2024-09-16T08:09:02.469 INFO:tasks.cephadm:Deploying osd.3 on smithi028 with /dev/vg_nvme/lv_1... 2024-09-16T08:09:02.470 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5622b2fe-7402-11ef-bceb-c7b262605968 -- lvm zap /dev/vg_nvme/lv_1 2024-09-16T08:09:02.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:09:02 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' 2024-09-16T08:09:02.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:09:02 smithi144 ceph-mon[25668]: from='osd.2 [v2:172.21.15.28:6818/3138093499,v1:172.21.15.28:6819/3138093499]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-09-16T08:09:03.444 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:09:03 smithi144 ceph-mon[25668]: pgmap v63: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-16T08:09:03.444 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:09:03 smithi144 ceph-mon[25668]: from='osd.2 [v2:172.21.15.28:6818/3138093499,v1:172.21.15.28:6819/3138093499]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-09-16T08:09:03.445 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:09:03 smithi144 ceph-mon[25668]: osdmap e17: 3 total, 2 up, 3 in 2024-09-16T08:09:03.445 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:09:03 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-16T08:09:03.445 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:09:03 smithi144 ceph-mon[25668]: from='osd.2 [v2:172.21.15.28:6818/3138093499,v1:172.21.15.28:6819/3138093499]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi028", "root=default"]}]: dispatch 2024-09-16T08:09:03.446 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:09:03 smithi028 ceph-mon[23744]: pgmap v63: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-16T08:09:03.446 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:09:03 smithi028 ceph-mon[23744]: from='osd.2 [v2:172.21.15.28:6818/3138093499,v1:172.21.15.28:6819/3138093499]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-09-16T08:09:03.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:09:03 smithi028 ceph-mon[23744]: osdmap e17: 3 total, 2 up, 3 in 2024-09-16T08:09:03.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:09:03 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-16T08:09:03.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:09:03 smithi028 ceph-mon[23744]: from='osd.2 [v2:172.21.15.28:6818/3138093499,v1:172.21.15.28:6819/3138093499]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi028", "root=default"]}]: dispatch 2024-09-16T08:09:03.447 INFO:journalctl@ceph.osd.2.smithi028.stdout:Sep 16 08:09:03 smithi028 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-2[40979]: 2024-09-16T08:09:03.181+0000 7fed23a38700 -1 osd.2 0 waiting for initial osdmap 2024-09-16T08:09:03.447 INFO:journalctl@ceph.osd.2.smithi028.stdout:Sep 16 08:09:03 smithi028 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-2[40979]: 2024-09-16T08:09:03.185+0000 7fed1e3cf700 -1 osd.2 18 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-16T08:09:04.446 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:09:04 smithi028 ceph-mon[23744]: from='osd.2 [v2:172.21.15.28:6818/3138093499,v1:172.21.15.28:6819/3138093499]' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi028", "root=default"]}]': finished 2024-09-16T08:09:04.446 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:09:04 smithi028 ceph-mon[23744]: osdmap e18: 3 total, 2 up, 3 in 2024-09-16T08:09:04.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:09:04 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-16T08:09:04.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:09:04 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-16T08:09:04.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:09:04 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi028.azvyzy/mirror_snapshot_schedule"}]: dispatch 2024-09-16T08:09:04.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:09:04 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi028.azvyzy/trash_purge_schedule"}]: dispatch 2024-09-16T08:09:04.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:09:04 smithi144 ceph-mon[25668]: from='osd.2 [v2:172.21.15.28:6818/3138093499,v1:172.21.15.28:6819/3138093499]' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi028", "root=default"]}]': finished 2024-09-16T08:09:04.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:09:04 smithi144 ceph-mon[25668]: osdmap e18: 3 total, 2 up, 3 in 2024-09-16T08:09:04.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:09:04 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-16T08:09:04.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:09:04 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-16T08:09:04.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:09:04 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi028.azvyzy/mirror_snapshot_schedule"}]: dispatch 2024-09-16T08:09:04.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:09:04 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi028.azvyzy/trash_purge_schedule"}]: dispatch 2024-09-16T08:09:05.283 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:09:05 smithi028 ceph-mon[23744]: purged_snaps scrub starts 2024-09-16T08:09:05.283 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:09:05 smithi028 ceph-mon[23744]: purged_snaps scrub ok 2024-09-16T08:09:05.283 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:09:05 smithi028 ceph-mon[23744]: pgmap v66: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-16T08:09:05.283 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:09:05 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-16T08:09:05.283 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:09:05 smithi028 ceph-mon[23744]: osd.2 [v2:172.21.15.28:6818/3138093499,v1:172.21.15.28:6819/3138093499] boot 2024-09-16T08:09:05.283 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:09:05 smithi028 ceph-mon[23744]: osdmap e19: 3 total, 3 up, 3 in 2024-09-16T08:09:05.284 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:09:05 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-16T08:09:05.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:09:05 smithi144 ceph-mon[25668]: purged_snaps scrub starts 2024-09-16T08:09:05.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:09:05 smithi144 ceph-mon[25668]: purged_snaps scrub ok 2024-09-16T08:09:05.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:09:05 smithi144 ceph-mon[25668]: pgmap v66: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-16T08:09:05.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:09:05 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-16T08:09:05.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:09:05 smithi144 ceph-mon[25668]: osd.2 [v2:172.21.15.28:6818/3138093499,v1:172.21.15.28:6819/3138093499] boot 2024-09-16T08:09:05.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:09:05 smithi144 ceph-mon[25668]: osdmap e19: 3 total, 3 up, 3 in 2024-09-16T08:09:05.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:09:05 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-16T08:09:06.446 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:09:06 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2024-09-16T08:09:06.446 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:09:06 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2024-09-16T08:09:06.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:09:06 smithi028 ceph-mon[23744]: osdmap e20: 3 total, 3 up, 3 in 2024-09-16T08:09:06.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:09:06 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' 2024-09-16T08:09:06.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:09:06 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-09-16T08:09:06.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:09:06 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2024-09-16T08:09:06.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:09:06 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2024-09-16T08:09:06.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:09:06 smithi144 ceph-mon[25668]: osdmap e20: 3 total, 3 up, 3 in 2024-09-16T08:09:06.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:09:06 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' 2024-09-16T08:09:06.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:09:06 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-09-16T08:09:06.818 INFO:teuthology.orchestra.run.smithi028.stdout: 2024-09-16T08:09:06.838 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5622b2fe-7402-11ef-bceb-c7b262605968 -- ceph orch daemon add osd smithi028:vg_nvme/lv_1 2024-09-16T08:09:07.401 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:09:07 smithi028 ceph-mon[23744]: pgmap v68: 0 pgs: ; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-09-16T08:09:07.401 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:09:07 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2024-09-16T08:09:07.402 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:09:07 smithi028 ceph-mon[23744]: osdmap e21: 3 total, 3 up, 3 in 2024-09-16T08:09:07.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:09:07 smithi144 ceph-mon[25668]: pgmap v68: 0 pgs: ; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-09-16T08:09:07.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:09:07 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2024-09-16T08:09:07.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:09:07 smithi144 ceph-mon[25668]: osdmap e21: 3 total, 3 up, 3 in 2024-09-16T08:09:08.649 INFO:journalctl@ceph.osd.0.smithi028.stdout:Sep 16 08:09:08 smithi028 sudo[42508]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-09-16T08:09:08.649 INFO:journalctl@ceph.osd.0.smithi028.stdout:Sep 16 08:09:08 smithi028 sudo[42508]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-16T08:09:08.649 INFO:journalctl@ceph.osd.0.smithi028.stdout:Sep 16 08:09:08 smithi028 sudo[42508]: pam_unix(sudo:session): session closed for user root 2024-09-16T08:09:09.196 INFO:journalctl@ceph.osd.0.smithi028.stdout:Sep 16 08:09:09 smithi028 sudo[42572]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-09-16T08:09:09.196 INFO:journalctl@ceph.osd.0.smithi028.stdout:Sep 16 08:09:09 smithi028 sudo[42572]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-16T08:09:09.196 INFO:journalctl@ceph.osd.0.smithi028.stdout:Sep 16 08:09:09 smithi028 sudo[42572]: pam_unix(sudo:session): session closed for user root 2024-09-16T08:09:09.446 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:09:09 smithi028 ceph-mon[23744]: pgmap v71: 1 pgs: 1 creating+peering; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-09-16T08:09:09.446 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:09:09 smithi028 ceph-mon[23744]: osdmap e22: 3 total, 3 up, 3 in 2024-09-16T08:09:09.446 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:09:09 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-16T08:09:09.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:09:09 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-16T08:09:09.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:09:09 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T08:09:09.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:09:09 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' 2024-09-16T08:09:09.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:09:09 smithi144 ceph-mon[25668]: pgmap v71: 1 pgs: 1 creating+peering; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-09-16T08:09:09.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:09:09 smithi144 ceph-mon[25668]: osdmap e22: 3 total, 3 up, 3 in 2024-09-16T08:09:09.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:09:09 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-16T08:09:09.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:09:09 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-16T08:09:09.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:09:09 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T08:09:09.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:09:09 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' 2024-09-16T08:09:09.771 INFO:journalctl@ceph.osd.1.smithi028.stdout:Sep 16 08:09:09 smithi028 sudo[42634]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-09-16T08:09:09.771 INFO:journalctl@ceph.osd.1.smithi028.stdout:Sep 16 08:09:09 smithi028 sudo[42634]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-16T08:09:09.771 INFO:journalctl@ceph.osd.1.smithi028.stdout:Sep 16 08:09:09 smithi028 sudo[42634]: pam_unix(sudo:session): session closed for user root 2024-09-16T08:09:10.196 INFO:journalctl@ceph.osd.1.smithi028.stdout:Sep 16 08:09:09 smithi028 sudo[42660]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-09-16T08:09:10.196 INFO:journalctl@ceph.osd.1.smithi028.stdout:Sep 16 08:09:09 smithi028 sudo[42660]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-16T08:09:10.196 INFO:journalctl@ceph.osd.1.smithi028.stdout:Sep 16 08:09:09 smithi028 sudo[42660]: pam_unix(sudo:session): session closed for user root 2024-09-16T08:09:10.562 INFO:journalctl@ceph.osd.2.smithi028.stdout:Sep 16 08:09:10 smithi028 sudo[42666]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-09-16T08:09:10.562 INFO:journalctl@ceph.osd.2.smithi028.stdout:Sep 16 08:09:10 smithi028 sudo[42666]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-16T08:09:10.562 INFO:journalctl@ceph.osd.2.smithi028.stdout:Sep 16 08:09:10 smithi028 sudo[42666]: pam_unix(sudo:session): session closed for user root 2024-09-16T08:09:10.563 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:09:10 smithi028 ceph-mon[23744]: from='client.14288 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi028:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T08:09:10.563 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:09:10 smithi028 ceph-mon[23744]: pgmap v73: 1 pgs: 1 creating+peering; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-09-16T08:09:10.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:09:10 smithi144 ceph-mon[25668]: from='client.14288 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi028:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T08:09:10.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:09:10 smithi144 ceph-mon[25668]: pgmap v73: 1 pgs: 1 creating+peering; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-09-16T08:09:11.196 INFO:journalctl@ceph.osd.2.smithi028.stdout:Sep 16 08:09:10 smithi028 sudo[42694]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-09-16T08:09:11.196 INFO:journalctl@ceph.osd.2.smithi028.stdout:Sep 16 08:09:10 smithi028 sudo[42694]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-16T08:09:11.197 INFO:journalctl@ceph.osd.2.smithi028.stdout:Sep 16 08:09:10 smithi028 sudo[42694]: pam_unix(sudo:session): session closed for user root 2024-09-16T08:09:11.696 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:09:11 smithi028 sudo[42722]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-09-16T08:09:11.696 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:09:11 smithi028 sudo[42722]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-16T08:09:11.696 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:09:11 smithi028 sudo[42722]: pam_unix(sudo:session): session closed for user root 2024-09-16T08:09:12.005 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:09:11 smithi144 ceph-mon[25668]: mgrmap e19: smithi028.azvyzy(active, since 2m), standbys: smithi144.utzszj 2024-09-16T08:09:12.005 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:09:11 smithi144 ceph-mon[25668]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-16T08:09:12.060 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:09:11 smithi028 sudo[42729]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-09-16T08:09:12.061 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:09:11 smithi028 sudo[42729]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-16T08:09:12.061 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:09:11 smithi028 sudo[42729]: pam_unix(sudo:session): session closed for user root 2024-09-16T08:09:12.061 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:09:11 smithi028 ceph-mon[23744]: mgrmap e19: smithi028.azvyzy(active, since 2m), standbys: smithi144.utzszj 2024-09-16T08:09:12.061 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:09:11 smithi028 ceph-mon[23744]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-16T08:09:12.258 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:09:12 smithi144 sudo[27887]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-09-16T08:09:12.258 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:09:12 smithi144 sudo[27887]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-16T08:09:12.258 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:09:12 smithi144 sudo[27887]: pam_unix(sudo:session): session closed for user root 2024-09-16T08:09:12.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:09:12 smithi144 sudo[27890]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-09-16T08:09:12.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:09:12 smithi144 sudo[27890]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-16T08:09:12.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:09:12 smithi144 sudo[27890]: pam_unix(sudo:session): session closed for user root 2024-09-16T08:09:13.081 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:09:12 smithi144 ceph-mon[25668]: pgmap v74: 1 pgs: 1 creating+peering; 0 B data, 16 MiB used, 268 GiB / 268 GiB avail 2024-09-16T08:09:13.081 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:09:12 smithi144 ceph-mon[25668]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-16T08:09:13.081 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:09:12 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "mon metadata", "id": "smithi028"}]: dispatch 2024-09-16T08:09:13.081 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:09:12 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "mon metadata", "id": "smithi144"}]: dispatch 2024-09-16T08:09:13.082 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:09:12 smithi144 ceph-mon[25668]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-16T08:09:13.082 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:09:12 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "mon metadata", "id": "smithi028"}]: dispatch 2024-09-16T08:09:13.082 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:09:12 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "mon metadata", "id": "smithi144"}]: dispatch 2024-09-16T08:09:13.082 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:09:12 smithi144 ceph-mon[25668]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-16T08:09:13.082 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:09:12 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' 2024-09-16T08:09:13.082 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:09:12 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-16T08:09:13.082 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:09:12 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-16T08:09:13.082 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:09:12 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-16T08:09:13.082 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:09:12 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' 2024-09-16T08:09:13.082 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:09:12 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' 2024-09-16T08:09:13.082 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:09:12 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' 2024-09-16T08:09:13.082 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:09:12 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-16T08:09:13.082 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:09:12 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T08:09:13.082 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:09:12 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-16T08:09:13.106 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:09:12 smithi028 ceph-mon[23744]: pgmap v74: 1 pgs: 1 creating+peering; 0 B data, 16 MiB used, 268 GiB / 268 GiB avail 2024-09-16T08:09:13.106 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:09:12 smithi028 ceph-mon[23744]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-16T08:09:13.106 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:09:12 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "mon metadata", "id": "smithi028"}]: dispatch 2024-09-16T08:09:13.106 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:09:12 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "mon metadata", "id": "smithi144"}]: dispatch 2024-09-16T08:09:13.106 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:09:12 smithi028 ceph-mon[23744]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-16T08:09:13.107 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:09:12 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "mon metadata", "id": "smithi028"}]: dispatch 2024-09-16T08:09:13.107 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:09:12 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "mon metadata", "id": "smithi144"}]: dispatch 2024-09-16T08:09:13.107 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:09:12 smithi028 ceph-mon[23744]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-16T08:09:13.107 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:09:12 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' 2024-09-16T08:09:13.107 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:09:12 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-16T08:09:13.107 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:09:12 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-16T08:09:13.107 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:09:12 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-16T08:09:13.107 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:09:12 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' 2024-09-16T08:09:13.107 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:09:12 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' 2024-09-16T08:09:13.107 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:09:12 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' 2024-09-16T08:09:13.107 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:09:12 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-16T08:09:13.107 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:09:12 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T08:09:13.107 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:09:12 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-16T08:09:13.946 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:09:13 smithi028 ceph-mon[23744]: Adjusting osd_memory_target on smithi028 to 4289M 2024-09-16T08:09:13.947 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:09:13 smithi028 ceph-mon[23744]: from='client.? 172.21.15.28:0/2832951745' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "2400d76b-78e4-43dd-becc-d1f80427f1df"}]: dispatch 2024-09-16T08:09:13.947 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:09:13 smithi028 ceph-mon[23744]: from='client.? 172.21.15.28:0/2832951745' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "2400d76b-78e4-43dd-becc-d1f80427f1df"}]': finished 2024-09-16T08:09:13.947 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:09:13 smithi028 ceph-mon[23744]: osdmap e23: 4 total, 3 up, 4 in 2024-09-16T08:09:13.947 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:09:13 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-16T08:09:13.947 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:09:13 smithi028 ceph-mon[23744]: from='client.? 172.21.15.28:0/403004061' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-16T08:09:14.081 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:09:13 smithi144 ceph-mon[25668]: Adjusting osd_memory_target on smithi028 to 4289M 2024-09-16T08:09:14.081 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:09:13 smithi144 ceph-mon[25668]: from='client.? 172.21.15.28:0/2832951745' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "2400d76b-78e4-43dd-becc-d1f80427f1df"}]: dispatch 2024-09-16T08:09:14.081 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:09:13 smithi144 ceph-mon[25668]: from='client.? 172.21.15.28:0/2832951745' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "2400d76b-78e4-43dd-becc-d1f80427f1df"}]': finished 2024-09-16T08:09:14.081 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:09:13 smithi144 ceph-mon[25668]: osdmap e23: 4 total, 3 up, 4 in 2024-09-16T08:09:14.081 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:09:13 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-16T08:09:14.081 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:09:13 smithi144 ceph-mon[25668]: from='client.? 172.21.15.28:0/403004061' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-16T08:09:15.146 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:09:14 smithi028 ceph-mon[23744]: pgmap v76: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-09-16T08:09:15.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:09:14 smithi144 ceph-mon[25668]: pgmap v76: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-09-16T08:09:17.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:09:17 smithi144 ceph-mon[25668]: pgmap v77: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-09-16T08:09:17.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:09:17 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' 2024-09-16T08:09:17.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:09:17 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' 2024-09-16T08:09:17.369 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:09:17 smithi028 ceph-mon[23744]: pgmap v77: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-09-16T08:09:17.369 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:09:17 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' 2024-09-16T08:09:17.369 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:09:17 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' 2024-09-16T08:09:19.127 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:09:19 smithi028 ceph-mon[23744]: pgmap v78: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 3.8 KiB/s rd, 165 KiB/s wr, 11 op/s 2024-09-16T08:09:19.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:09:19 smithi144 ceph-mon[25668]: pgmap v78: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 3.8 KiB/s rd, 165 KiB/s wr, 11 op/s 2024-09-16T08:09:20.097 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:09:20 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-09-16T08:09:20.098 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:09:20 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T08:09:20.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:09:20 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-09-16T08:09:20.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:09:20 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T08:09:21.078 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:09:21 smithi028 ceph-mon[23744]: Deploying daemon osd.3 on smithi028 2024-09-16T08:09:21.079 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:09:21 smithi028 ceph-mon[23744]: pgmap v79: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 2.7 KiB/s rd, 116 KiB/s wr, 8 op/s 2024-09-16T08:09:21.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:09:21 smithi144 ceph-mon[25668]: Deploying daemon osd.3 on smithi028 2024-09-16T08:09:21.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:09:21 smithi144 ceph-mon[25668]: pgmap v79: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 2.7 KiB/s rd, 116 KiB/s wr, 8 op/s 2024-09-16T08:09:23.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:09:23 smithi144 ceph-mon[25668]: pgmap v80: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 2.1 KiB/s rd, 90 KiB/s wr, 6 op/s 2024-09-16T08:09:23.332 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:09:23 smithi028 ceph-mon[23744]: pgmap v80: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 2.1 KiB/s rd, 90 KiB/s wr, 6 op/s 2024-09-16T08:09:25.195 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:09:25 smithi028 ceph-mon[23744]: pgmap v81: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 1.7 KiB/s rd, 73 KiB/s wr, 5 op/s 2024-09-16T08:09:25.195 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:09:25 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' 2024-09-16T08:09:25.196 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:09:25 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-16T08:09:25.196 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:09:25 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T08:09:25.196 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:09:25 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-16T08:09:25.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:09:25 smithi144 ceph-mon[25668]: pgmap v81: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 1.7 KiB/s rd, 73 KiB/s wr, 5 op/s 2024-09-16T08:09:25.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:09:25 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' 2024-09-16T08:09:25.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:09:25 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-16T08:09:25.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:09:25 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T08:09:25.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:09:25 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-16T08:09:27.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:09:27 smithi144 ceph-mon[25668]: pgmap v82: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2024-09-16T08:09:27.446 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:09:27 smithi028 ceph-mon[23744]: pgmap v82: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2024-09-16T08:09:27.741 INFO:teuthology.orchestra.run.smithi028.stdout:Created osd(s) 3 on host 'smithi028' 2024-09-16T08:09:28.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:09:28 smithi144 ceph-mon[25668]: from='osd.3 [v2:172.21.15.28:6826/3665954147,v1:172.21.15.28:6827/3665954147]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-09-16T08:09:28.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:09:28 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' 2024-09-16T08:09:28.346 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:09:28 smithi028 ceph-mon[23744]: from='osd.3 [v2:172.21.15.28:6826/3665954147,v1:172.21.15.28:6827/3665954147]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-09-16T08:09:28.346 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:09:28 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' 2024-09-16T08:09:28.372 DEBUG:teuthology.orchestra.run.smithi028:osd.3> sudo journalctl -f -n 0 -u ceph-5622b2fe-7402-11ef-bceb-c7b262605968@osd.3.service 2024-09-16T08:09:28.374 INFO:tasks.cephadm:Deploying osd.4 on smithi144 with /dev/vg_nvme/lv_4... 2024-09-16T08:09:28.374 DEBUG:teuthology.orchestra.run.smithi144:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5622b2fe-7402-11ef-bceb-c7b262605968 -- lvm zap /dev/vg_nvme/lv_4 2024-09-16T08:09:29.226 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:09:29 smithi028 ceph-mon[23744]: pgmap v83: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2024-09-16T08:09:29.226 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:09:29 smithi028 ceph-mon[23744]: from='osd.3 [v2:172.21.15.28:6826/3665954147,v1:172.21.15.28:6827/3665954147]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-09-16T08:09:29.226 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:09:29 smithi028 ceph-mon[23744]: osdmap e24: 4 total, 3 up, 4 in 2024-09-16T08:09:29.226 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:09:29 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-16T08:09:29.227 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:09:29 smithi028 ceph-mon[23744]: from='osd.3 [v2:172.21.15.28:6826/3665954147,v1:172.21.15.28:6827/3665954147]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi028", "root=default"]}]: dispatch 2024-09-16T08:09:29.227 INFO:journalctl@ceph.osd.3.smithi028.stdout:Sep 16 08:09:29 smithi028 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-3[44733]: 2024-09-16T08:09:29.070+0000 7ff56efb4700 -1 osd.3 0 waiting for initial osdmap 2024-09-16T08:09:29.227 INFO:journalctl@ceph.osd.3.smithi028.stdout:Sep 16 08:09:29 smithi028 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-3[44733]: 2024-09-16T08:09:29.078+0000 7ff56a94d700 -1 osd.3 25 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-16T08:09:29.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:09:29 smithi144 ceph-mon[25668]: pgmap v83: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2024-09-16T08:09:29.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:09:29 smithi144 ceph-mon[25668]: from='osd.3 [v2:172.21.15.28:6826/3665954147,v1:172.21.15.28:6827/3665954147]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-09-16T08:09:29.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:09:29 smithi144 ceph-mon[25668]: osdmap e24: 4 total, 3 up, 4 in 2024-09-16T08:09:29.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:09:29 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-16T08:09:29.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:09:29 smithi144 ceph-mon[25668]: from='osd.3 [v2:172.21.15.28:6826/3665954147,v1:172.21.15.28:6827/3665954147]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi028", "root=default"]}]: dispatch 2024-09-16T08:09:30.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:09:30 smithi144 ceph-mon[25668]: from='osd.3 [v2:172.21.15.28:6826/3665954147,v1:172.21.15.28:6827/3665954147]' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi028", "root=default"]}]': finished 2024-09-16T08:09:30.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:09:30 smithi144 ceph-mon[25668]: osdmap e25: 4 total, 3 up, 4 in 2024-09-16T08:09:30.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:09:30 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-16T08:09:30.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:09:30 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-16T08:09:30.406 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:09:30 smithi028 ceph-mon[23744]: from='osd.3 [v2:172.21.15.28:6826/3665954147,v1:172.21.15.28:6827/3665954147]' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi028", "root=default"]}]': finished 2024-09-16T08:09:30.406 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:09:30 smithi028 ceph-mon[23744]: osdmap e25: 4 total, 3 up, 4 in 2024-09-16T08:09:30.406 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:09:30 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-16T08:09:30.406 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:09:30 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-16T08:09:30.968 INFO:teuthology.orchestra.run.smithi144.stdout: 2024-09-16T08:09:30.987 DEBUG:teuthology.orchestra.run.smithi144:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5622b2fe-7402-11ef-bceb-c7b262605968 -- ceph orch daemon add osd smithi144:vg_nvme/lv_4 2024-09-16T08:09:31.285 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:09:31 smithi144 ceph-mon[25668]: purged_snaps scrub starts 2024-09-16T08:09:31.285 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:09:31 smithi144 ceph-mon[25668]: purged_snaps scrub ok 2024-09-16T08:09:31.285 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:09:31 smithi144 ceph-mon[25668]: pgmap v86: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-09-16T08:09:31.285 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:09:31 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-16T08:09:31.285 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:09:31 smithi144 ceph-mon[25668]: osd.3 [v2:172.21.15.28:6826/3665954147,v1:172.21.15.28:6827/3665954147] boot 2024-09-16T08:09:31.285 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:09:31 smithi144 ceph-mon[25668]: osdmap e26: 4 total, 4 up, 4 in 2024-09-16T08:09:31.285 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:09:31 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-16T08:09:31.285 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:09:31 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' 2024-09-16T08:09:31.288 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:09:31 smithi028 ceph-mon[23744]: purged_snaps scrub starts 2024-09-16T08:09:31.288 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:09:31 smithi028 ceph-mon[23744]: purged_snaps scrub ok 2024-09-16T08:09:31.288 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:09:31 smithi028 ceph-mon[23744]: pgmap v86: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-09-16T08:09:31.288 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:09:31 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-16T08:09:31.289 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:09:31 smithi028 ceph-mon[23744]: osd.3 [v2:172.21.15.28:6826/3665954147,v1:172.21.15.28:6827/3665954147] boot 2024-09-16T08:09:31.289 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:09:31 smithi028 ceph-mon[23744]: osdmap e26: 4 total, 4 up, 4 in 2024-09-16T08:09:31.289 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:09:31 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-16T08:09:31.289 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:09:31 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' 2024-09-16T08:09:32.133 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:09:32 smithi144 ceph-mon[25668]: osdmap e27: 4 total, 4 up, 4 in 2024-09-16T08:09:32.372 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:09:32 smithi028 ceph-mon[23744]: osdmap e27: 4 total, 4 up, 4 in 2024-09-16T08:09:33.166 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:09:33 smithi144 ceph-mon[25668]: pgmap v89: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-16T08:09:33.167 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:09:33 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-16T08:09:33.167 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:09:33 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-16T08:09:33.167 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:09:33 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T08:09:33.440 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:09:33 smithi028 ceph-mon[23744]: pgmap v89: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-16T08:09:33.440 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:09:33 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-16T08:09:33.441 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:09:33 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-16T08:09:33.441 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:09:33 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T08:09:34.314 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:09:34 smithi144 ceph-mon[25668]: from='client.24157 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi144:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T08:09:34.314 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:09:34 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' 2024-09-16T08:09:34.314 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:09:34 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-16T08:09:34.314 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:09:34 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-16T08:09:34.315 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:09:34 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-16T08:09:34.315 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:09:34 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-16T08:09:34.315 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:09:34 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' 2024-09-16T08:09:34.315 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:09:34 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' 2024-09-16T08:09:34.446 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:09:34 smithi028 ceph-mon[23744]: from='client.24157 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi144:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T08:09:34.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:09:34 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' 2024-09-16T08:09:34.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:09:34 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-16T08:09:34.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:09:34 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-16T08:09:34.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:09:34 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-16T08:09:34.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:09:34 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-16T08:09:34.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:09:34 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' 2024-09-16T08:09:34.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:09:34 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' 2024-09-16T08:09:35.437 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:09:35 smithi028 ceph-mon[23744]: pgmap v90: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-16T08:09:35.437 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:09:35 smithi028 ceph-mon[23744]: Detected new or changed devices on smithi028 2024-09-16T08:09:35.437 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:09:35 smithi028 ceph-mon[23744]: Adjusting osd_memory_target on smithi028 to 3217M 2024-09-16T08:09:35.437 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:09:35 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-16T08:09:35.437 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:09:35 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T08:09:35.437 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:09:35 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-16T08:09:35.437 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:09:35 smithi028 ceph-mon[23744]: from='client.? 172.21.15.144:0/3658437065' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a929d7e2-fac0-47fe-9919-a4329c008de2"}]: dispatch 2024-09-16T08:09:35.437 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:09:35 smithi028 ceph-mon[23744]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a929d7e2-fac0-47fe-9919-a4329c008de2"}]: dispatch 2024-09-16T08:09:35.437 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:09:35 smithi028 ceph-mon[23744]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "a929d7e2-fac0-47fe-9919-a4329c008de2"}]': finished 2024-09-16T08:09:35.437 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:09:35 smithi028 ceph-mon[23744]: osdmap e28: 5 total, 4 up, 5 in 2024-09-16T08:09:35.437 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:09:35 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-16T08:09:35.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:09:35 smithi144 ceph-mon[25668]: pgmap v90: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-16T08:09:35.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:09:35 smithi144 ceph-mon[25668]: Detected new or changed devices on smithi028 2024-09-16T08:09:35.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:09:35 smithi144 ceph-mon[25668]: Adjusting osd_memory_target on smithi028 to 3217M 2024-09-16T08:09:35.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:09:35 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-16T08:09:35.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:09:35 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T08:09:35.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:09:35 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-16T08:09:35.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:09:35 smithi144 ceph-mon[25668]: from='client.? 172.21.15.144:0/3658437065' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a929d7e2-fac0-47fe-9919-a4329c008de2"}]: dispatch 2024-09-16T08:09:35.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:09:35 smithi144 ceph-mon[25668]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a929d7e2-fac0-47fe-9919-a4329c008de2"}]: dispatch 2024-09-16T08:09:35.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:09:35 smithi144 ceph-mon[25668]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "a929d7e2-fac0-47fe-9919-a4329c008de2"}]': finished 2024-09-16T08:09:35.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:09:35 smithi144 ceph-mon[25668]: osdmap e28: 5 total, 4 up, 5 in 2024-09-16T08:09:35.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:09:35 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-16T08:09:36.446 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:09:36 smithi028 ceph-mon[23744]: from='client.? 172.21.15.144:0/1158613393' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-16T08:09:36.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:09:36 smithi144 ceph-mon[25668]: from='client.? 172.21.15.144:0/1158613393' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-16T08:09:37.392 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:09:37 smithi028 ceph-mon[23744]: pgmap v92: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-16T08:09:37.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:09:37 smithi144 ceph-mon[25668]: pgmap v92: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-16T08:09:39.081 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:09:39 smithi144 ceph-mon[25668]: pgmap v93: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-16T08:09:39.081 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:09:39 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' 2024-09-16T08:09:39.081 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:09:39 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' 2024-09-16T08:09:39.446 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:09:39 smithi028 ceph-mon[23744]: pgmap v93: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-16T08:09:39.446 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:09:39 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' 2024-09-16T08:09:39.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:09:39 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' 2024-09-16T08:09:41.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:09:41 smithi144 ceph-mon[25668]: pgmap v94: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-16T08:09:41.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:09:41 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-09-16T08:09:41.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:09:41 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T08:09:41.446 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:09:41 smithi028 ceph-mon[23744]: pgmap v94: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-16T08:09:41.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:09:41 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-09-16T08:09:41.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:09:41 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T08:09:42.319 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:09:42 smithi144 ceph-mon[25668]: Deploying daemon osd.4 on smithi144 2024-09-16T08:09:42.364 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:09:42 smithi028 ceph-mon[23744]: Deploying daemon osd.4 on smithi144 2024-09-16T08:09:43.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:09:43 smithi144 ceph-mon[25668]: pgmap v95: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-16T08:09:43.446 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:09:43 smithi028 ceph-mon[23744]: pgmap v95: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-16T08:09:45.149 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:09:45 smithi144 ceph-mon[25668]: pgmap v96: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-16T08:09:45.446 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:09:45 smithi028 ceph-mon[23744]: pgmap v96: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-16T08:09:47.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:09:47 smithi144 ceph-mon[25668]: pgmap v97: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-16T08:09:47.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:09:47 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' 2024-09-16T08:09:47.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:09:47 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-16T08:09:47.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:09:47 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T08:09:47.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:09:47 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-16T08:09:47.446 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:09:47 smithi028 ceph-mon[23744]: pgmap v97: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-16T08:09:47.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:09:47 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' 2024-09-16T08:09:47.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:09:47 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-16T08:09:47.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:09:47 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T08:09:47.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:09:47 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-16T08:09:49.319 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:09:49 smithi144 ceph-mon[25668]: pgmap v98: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-16T08:09:49.446 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:09:49 smithi028 ceph-mon[23744]: pgmap v98: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-16T08:09:49.711 INFO:teuthology.orchestra.run.smithi144.stdout:Created osd(s) 4 on host 'smithi144' 2024-09-16T08:09:50.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:09:50 smithi144 ceph-mon[25668]: from='osd.4 [v2:172.21.15.144:6800/1792696254,v1:172.21.15.144:6801/1792696254]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-09-16T08:09:50.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:09:50 smithi144 ceph-mon[25668]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-09-16T08:09:50.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:09:50 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' 2024-09-16T08:09:50.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:09:50 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' 2024-09-16T08:09:50.406 DEBUG:teuthology.orchestra.run.smithi144:osd.4> sudo journalctl -f -n 0 -u ceph-5622b2fe-7402-11ef-bceb-c7b262605968@osd.4.service 2024-09-16T08:09:50.409 INFO:tasks.cephadm:Deploying osd.5 on smithi144 with /dev/vg_nvme/lv_3... 2024-09-16T08:09:50.409 DEBUG:teuthology.orchestra.run.smithi144:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5622b2fe-7402-11ef-bceb-c7b262605968 -- lvm zap /dev/vg_nvme/lv_3 2024-09-16T08:09:50.446 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:09:50 smithi028 ceph-mon[23744]: from='osd.4 [v2:172.21.15.144:6800/1792696254,v1:172.21.15.144:6801/1792696254]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-09-16T08:09:50.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:09:50 smithi028 ceph-mon[23744]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-09-16T08:09:50.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:09:50 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' 2024-09-16T08:09:50.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:09:50 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' 2024-09-16T08:09:51.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:09:51 smithi144 ceph-mon[25668]: pgmap v99: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-16T08:09:51.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:09:51 smithi144 ceph-mon[25668]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-09-16T08:09:51.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:09:51 smithi144 ceph-mon[25668]: osdmap e29: 5 total, 4 up, 5 in 2024-09-16T08:09:51.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:09:51 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-16T08:09:51.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:09:51 smithi144 ceph-mon[25668]: from='osd.4 [v2:172.21.15.144:6800/1792696254,v1:172.21.15.144:6801/1792696254]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi144", "root=default"]}]: dispatch 2024-09-16T08:09:51.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:09:51 smithi144 ceph-mon[25668]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi144", "root=default"]}]: dispatch 2024-09-16T08:09:51.332 INFO:journalctl@ceph.osd.4.smithi144.stdout:Sep 16 08:09:51 smithi144 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-4[29728]: 2024-09-16T08:09:51.068+0000 7fd17f5a5700 -1 osd.4 0 waiting for initial osdmap 2024-09-16T08:09:51.332 INFO:journalctl@ceph.osd.4.smithi144.stdout:Sep 16 08:09:51 smithi144 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-4[29728]: 2024-09-16T08:09:51.072+0000 7fd179f3c700 -1 osd.4 30 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-16T08:09:51.446 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:09:51 smithi028 ceph-mon[23744]: pgmap v99: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-16T08:09:51.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:09:51 smithi028 ceph-mon[23744]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-09-16T08:09:51.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:09:51 smithi028 ceph-mon[23744]: osdmap e29: 5 total, 4 up, 5 in 2024-09-16T08:09:51.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:09:51 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-16T08:09:51.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:09:51 smithi028 ceph-mon[23744]: from='osd.4 [v2:172.21.15.144:6800/1792696254,v1:172.21.15.144:6801/1792696254]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi144", "root=default"]}]: dispatch 2024-09-16T08:09:51.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:09:51 smithi028 ceph-mon[23744]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi144", "root=default"]}]: dispatch 2024-09-16T08:09:52.081 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:09:52 smithi144 ceph-mon[25668]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi144", "root=default"]}]': finished 2024-09-16T08:09:52.081 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:09:52 smithi144 ceph-mon[25668]: osdmap e30: 5 total, 4 up, 5 in 2024-09-16T08:09:52.082 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:09:52 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-16T08:09:52.082 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:09:52 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-16T08:09:52.082 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:09:52 smithi144 ceph-mon[25668]: osd.4 [v2:172.21.15.144:6800/1792696254,v1:172.21.15.144:6801/1792696254] boot 2024-09-16T08:09:52.082 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:09:52 smithi144 ceph-mon[25668]: osdmap e31: 5 total, 5 up, 5 in 2024-09-16T08:09:52.082 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:09:52 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-16T08:09:52.446 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:09:52 smithi028 ceph-mon[23744]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi144", "root=default"]}]': finished 2024-09-16T08:09:52.446 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:09:52 smithi028 ceph-mon[23744]: osdmap e30: 5 total, 4 up, 5 in 2024-09-16T08:09:52.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:09:52 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-16T08:09:52.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:09:52 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-16T08:09:52.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:09:52 smithi028 ceph-mon[23744]: osd.4 [v2:172.21.15.144:6800/1792696254,v1:172.21.15.144:6801/1792696254] boot 2024-09-16T08:09:52.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:09:52 smithi028 ceph-mon[23744]: osdmap e31: 5 total, 5 up, 5 in 2024-09-16T08:09:52.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:09:52 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-16T08:09:53.242 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:09:53 smithi144 ceph-mon[25668]: purged_snaps scrub starts 2024-09-16T08:09:53.242 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:09:53 smithi144 ceph-mon[25668]: purged_snaps scrub ok 2024-09-16T08:09:53.242 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:09:53 smithi144 ceph-mon[25668]: pgmap v102: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-16T08:09:53.242 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:09:53 smithi144 ceph-mon[25668]: osdmap e32: 5 total, 5 up, 5 in 2024-09-16T08:09:53.446 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:09:53 smithi028 ceph-mon[23744]: purged_snaps scrub starts 2024-09-16T08:09:53.446 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:09:53 smithi028 ceph-mon[23744]: purged_snaps scrub ok 2024-09-16T08:09:53.446 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:09:53 smithi028 ceph-mon[23744]: pgmap v102: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-16T08:09:53.446 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:09:53 smithi028 ceph-mon[23744]: osdmap e32: 5 total, 5 up, 5 in 2024-09-16T08:09:54.510 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:09:54 smithi144 ceph-mon[25668]: Detected new or changed devices on smithi144 2024-09-16T08:09:54.510 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:09:54 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' 2024-09-16T08:09:54.510 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:09:54 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-16T08:09:54.510 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:09:54 smithi144 ceph-mon[25668]: Adjusting osd_memory_target on smithi144 to 15936M 2024-09-16T08:09:54.510 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:09:54 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' 2024-09-16T08:09:54.510 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:09:54 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' 2024-09-16T08:09:54.510 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:09:54 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-16T08:09:54.510 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:09:54 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T08:09:54.511 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:09:54 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-16T08:09:54.511 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:09:54 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' 2024-09-16T08:09:54.511 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:09:54 smithi144 ceph-mon[25668]: pgmap v105: 1 pgs: 1 remapped+peering; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail 2024-09-16T08:09:54.511 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:09:54 smithi144 ceph-mon[25668]: osdmap e33: 5 total, 5 up, 5 in 2024-09-16T08:09:54.697 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:09:54 smithi028 ceph-mon[23744]: Detected new or changed devices on smithi144 2024-09-16T08:09:54.698 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:09:54 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' 2024-09-16T08:09:54.698 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:09:54 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-16T08:09:54.698 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:09:54 smithi028 ceph-mon[23744]: Adjusting osd_memory_target on smithi144 to 15936M 2024-09-16T08:09:54.698 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:09:54 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' 2024-09-16T08:09:54.698 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:09:54 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' 2024-09-16T08:09:54.698 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:09:54 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-16T08:09:54.698 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:09:54 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T08:09:54.698 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:09:54 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-16T08:09:54.698 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:09:54 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' 2024-09-16T08:09:54.698 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:09:54 smithi028 ceph-mon[23744]: pgmap v105: 1 pgs: 1 remapped+peering; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail 2024-09-16T08:09:54.698 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:09:54 smithi028 ceph-mon[23744]: osdmap e33: 5 total, 5 up, 5 in 2024-09-16T08:09:55.049 INFO:teuthology.orchestra.run.smithi144.stdout: 2024-09-16T08:09:55.069 DEBUG:teuthology.orchestra.run.smithi144:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5622b2fe-7402-11ef-bceb-c7b262605968 -- ceph orch daemon add osd smithi144:vg_nvme/lv_3 2024-09-16T08:09:56.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:09:56 smithi144 ceph-mon[25668]: osdmap e34: 5 total, 5 up, 5 in 2024-09-16T08:09:56.446 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:09:56 smithi028 ceph-mon[23744]: osdmap e34: 5 total, 5 up, 5 in 2024-09-16T08:09:57.311 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:09:57 smithi144 ceph-mon[25668]: pgmap v108: 1 pgs: 1 remapped+peering; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail 2024-09-16T08:09:57.311 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:09:57 smithi144 ceph-mon[25668]: osdmap e35: 5 total, 5 up, 5 in 2024-09-16T08:09:57.312 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:09:57 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-16T08:09:57.312 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:09:57 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-16T08:09:57.312 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:09:57 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T08:09:57.446 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:09:57 smithi028 ceph-mon[23744]: pgmap v108: 1 pgs: 1 remapped+peering; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail 2024-09-16T08:09:57.446 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:09:57 smithi028 ceph-mon[23744]: osdmap e35: 5 total, 5 up, 5 in 2024-09-16T08:09:57.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:09:57 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-16T08:09:57.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:09:57 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-16T08:09:57.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:09:57 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T08:09:58.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:09:58 smithi144 ceph-mon[25668]: from='client.24175 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi144:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T08:09:58.446 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:09:58 smithi028 ceph-mon[23744]: from='client.24175 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi144:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T08:09:59.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:09:59 smithi144 ceph-mon[25668]: pgmap v110: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 113 KiB/s, 0 objects/s recovering 2024-09-16T08:09:59.446 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:09:59 smithi028 ceph-mon[23744]: pgmap v110: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 113 KiB/s, 0 objects/s recovering 2024-09-16T08:10:00.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:10:00 smithi144 ceph-mon[25668]: from='client.? 172.21.15.144:0/4249097397' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "3f1ee7e1-e3ee-4351-8492-3573e3cb0fea"}]: dispatch 2024-09-16T08:10:00.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:10:00 smithi144 ceph-mon[25668]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "3f1ee7e1-e3ee-4351-8492-3573e3cb0fea"}]: dispatch 2024-09-16T08:10:00.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:10:00 smithi144 ceph-mon[25668]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "3f1ee7e1-e3ee-4351-8492-3573e3cb0fea"}]': finished 2024-09-16T08:10:00.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:10:00 smithi144 ceph-mon[25668]: osdmap e36: 6 total, 5 up, 6 in 2024-09-16T08:10:00.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:10:00 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-16T08:10:00.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:10:00 smithi144 ceph-mon[25668]: from='client.? 172.21.15.144:0/2341178992' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-16T08:10:00.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:10:00 smithi144 ceph-mon[25668]: overall HEALTH_OK 2024-09-16T08:10:00.348 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:10:00 smithi028 ceph-mon[23744]: from='client.? 172.21.15.144:0/4249097397' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "3f1ee7e1-e3ee-4351-8492-3573e3cb0fea"}]: dispatch 2024-09-16T08:10:00.348 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:10:00 smithi028 ceph-mon[23744]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "3f1ee7e1-e3ee-4351-8492-3573e3cb0fea"}]: dispatch 2024-09-16T08:10:00.348 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:10:00 smithi028 ceph-mon[23744]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "3f1ee7e1-e3ee-4351-8492-3573e3cb0fea"}]': finished 2024-09-16T08:10:00.348 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:10:00 smithi028 ceph-mon[23744]: osdmap e36: 6 total, 5 up, 6 in 2024-09-16T08:10:00.348 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:10:00 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-16T08:10:00.348 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:10:00 smithi028 ceph-mon[23744]: from='client.? 172.21.15.144:0/2341178992' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-16T08:10:00.348 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:10:00 smithi028 ceph-mon[23744]: overall HEALTH_OK 2024-09-16T08:10:01.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:10:01 smithi144 ceph-mon[25668]: pgmap v112: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 93 KiB/s, 0 objects/s recovering 2024-09-16T08:10:01.446 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:10:01 smithi028 ceph-mon[23744]: pgmap v112: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 93 KiB/s, 0 objects/s recovering 2024-09-16T08:10:03.327 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:10:03 smithi144 ceph-mon[25668]: pgmap v113: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 79 KiB/s, 0 objects/s recovering 2024-09-16T08:10:03.446 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:10:03 smithi028 ceph-mon[23744]: pgmap v113: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 79 KiB/s, 0 objects/s recovering 2024-09-16T08:10:04.201 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:10:04 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi028.azvyzy/mirror_snapshot_schedule"}]: dispatch 2024-09-16T08:10:04.202 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:10:04 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi028.azvyzy/trash_purge_schedule"}]: dispatch 2024-09-16T08:10:04.446 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:10:04 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi028.azvyzy/mirror_snapshot_schedule"}]: dispatch 2024-09-16T08:10:04.446 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:10:04 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi028.azvyzy/trash_purge_schedule"}]: dispatch 2024-09-16T08:10:05.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:10:05 smithi144 ceph-mon[25668]: pgmap v114: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-09-16T08:10:05.446 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:10:05 smithi028 ceph-mon[23744]: pgmap v114: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-09-16T08:10:06.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:10:06 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-09-16T08:10:06.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:10:06 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T08:10:06.446 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:10:06 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-09-16T08:10:06.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:10:06 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T08:10:07.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:10:07 smithi144 ceph-mon[25668]: Deploying daemon osd.5 on smithi144 2024-09-16T08:10:07.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:10:07 smithi144 ceph-mon[25668]: pgmap v115: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 54 KiB/s, 0 objects/s recovering 2024-09-16T08:10:07.446 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:10:07 smithi028 ceph-mon[23744]: Deploying daemon osd.5 on smithi144 2024-09-16T08:10:07.446 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:10:07 smithi028 ceph-mon[23744]: pgmap v115: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 54 KiB/s, 0 objects/s recovering 2024-09-16T08:10:09.176 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:10:09 smithi144 ceph-mon[25668]: pgmap v116: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-16T08:10:09.446 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:10:09 smithi028 ceph-mon[23744]: pgmap v116: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-16T08:10:11.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:10:11 smithi144 ceph-mon[25668]: pgmap v117: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-16T08:10:11.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:10:11 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' 2024-09-16T08:10:11.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:10:11 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-16T08:10:11.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:10:11 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T08:10:11.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:10:11 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-16T08:10:11.446 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:10:11 smithi028 ceph-mon[23744]: pgmap v117: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-16T08:10:11.446 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:10:11 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' 2024-09-16T08:10:11.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:10:11 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-16T08:10:11.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:10:11 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T08:10:11.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:10:11 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-16T08:10:13.442 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:10:13 smithi144 ceph-mon[25668]: pgmap v118: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-16T08:10:13.446 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:10:13 smithi028 ceph-mon[23744]: pgmap v118: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-16T08:10:14.214 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:10:14 smithi144 ceph-mon[25668]: from='osd.5 [v2:172.21.15.144:6808/2187592584,v1:172.21.15.144:6809/2187592584]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-09-16T08:10:14.214 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:10:14 smithi144 ceph-mon[25668]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-09-16T08:10:14.245 INFO:teuthology.orchestra.run.smithi144.stdout:Created osd(s) 5 on host 'smithi144' 2024-09-16T08:10:14.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:10:14 smithi028 ceph-mon[23744]: from='osd.5 [v2:172.21.15.144:6808/2187592584,v1:172.21.15.144:6809/2187592584]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-09-16T08:10:14.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:10:14 smithi028 ceph-mon[23744]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-09-16T08:10:14.882 DEBUG:teuthology.orchestra.run.smithi144:osd.5> sudo journalctl -f -n 0 -u ceph-5622b2fe-7402-11ef-bceb-c7b262605968@osd.5.service 2024-09-16T08:10:14.884 INFO:tasks.cephadm:Deploying osd.6 on smithi144 with /dev/vg_nvme/lv_2... 2024-09-16T08:10:14.884 DEBUG:teuthology.orchestra.run.smithi144:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5622b2fe-7402-11ef-bceb-c7b262605968 -- lvm zap /dev/vg_nvme/lv_2 2024-09-16T08:10:15.109 INFO:journalctl@ceph.osd.5.smithi144.stdout:Sep 16 08:10:15 smithi144 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-5[32740]: 2024-09-16T08:10:15.103+0000 7f3f4615b700 -1 osd.5 0 waiting for initial osdmap 2024-09-16T08:10:15.109 INFO:journalctl@ceph.osd.5.smithi144.stdout:Sep 16 08:10:15 smithi144 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-5[32740]: 2024-09-16T08:10:15.109+0000 7f3f402f1700 -1 osd.5 38 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-16T08:10:15.110 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:10:15 smithi144 ceph-mon[25668]: pgmap v119: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-16T08:10:15.110 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:10:15 smithi144 ceph-mon[25668]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-09-16T08:10:15.110 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:10:15 smithi144 ceph-mon[25668]: osdmap e37: 6 total, 5 up, 6 in 2024-09-16T08:10:15.110 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:10:15 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-16T08:10:15.110 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:10:15 smithi144 ceph-mon[25668]: from='osd.5 [v2:172.21.15.144:6808/2187592584,v1:172.21.15.144:6809/2187592584]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi144", "root=default"]}]: dispatch 2024-09-16T08:10:15.110 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:10:15 smithi144 ceph-mon[25668]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi144", "root=default"]}]: dispatch 2024-09-16T08:10:15.110 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:10:15 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' 2024-09-16T08:10:15.110 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:10:15 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' 2024-09-16T08:10:15.446 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:10:15 smithi028 ceph-mon[23744]: pgmap v119: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-16T08:10:15.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:10:15 smithi028 ceph-mon[23744]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-09-16T08:10:15.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:10:15 smithi028 ceph-mon[23744]: osdmap e37: 6 total, 5 up, 6 in 2024-09-16T08:10:15.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:10:15 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-16T08:10:15.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:10:15 smithi028 ceph-mon[23744]: from='osd.5 [v2:172.21.15.144:6808/2187592584,v1:172.21.15.144:6809/2187592584]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi144", "root=default"]}]: dispatch 2024-09-16T08:10:15.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:10:15 smithi028 ceph-mon[23744]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi144", "root=default"]}]: dispatch 2024-09-16T08:10:15.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:10:15 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' 2024-09-16T08:10:15.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:10:15 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' 2024-09-16T08:10:16.207 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:10:16 smithi144 ceph-mon[25668]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi144", "root=default"]}]': finished 2024-09-16T08:10:16.207 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:10:16 smithi144 ceph-mon[25668]: osdmap e38: 6 total, 5 up, 6 in 2024-09-16T08:10:16.207 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:10:16 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-16T08:10:16.207 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:10:16 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-16T08:10:16.446 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:10:16 smithi028 ceph-mon[23744]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi144", "root=default"]}]': finished 2024-09-16T08:10:16.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:10:16 smithi028 ceph-mon[23744]: osdmap e38: 6 total, 5 up, 6 in 2024-09-16T08:10:16.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:10:16 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-16T08:10:16.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:10:16 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-16T08:10:17.363 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:10:17 smithi144 ceph-mon[25668]: purged_snaps scrub starts 2024-09-16T08:10:17.363 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:10:17 smithi144 ceph-mon[25668]: purged_snaps scrub ok 2024-09-16T08:10:17.363 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:10:17 smithi144 ceph-mon[25668]: pgmap v122: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-16T08:10:17.364 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:10:17 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-16T08:10:17.364 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:10:17 smithi144 ceph-mon[25668]: osd.5 [v2:172.21.15.144:6808/2187592584,v1:172.21.15.144:6809/2187592584] boot 2024-09-16T08:10:17.364 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:10:17 smithi144 ceph-mon[25668]: osdmap e39: 6 total, 6 up, 6 in 2024-09-16T08:10:17.364 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:10:17 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-16T08:10:17.446 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:10:17 smithi028 ceph-mon[23744]: purged_snaps scrub starts 2024-09-16T08:10:17.446 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:10:17 smithi028 ceph-mon[23744]: purged_snaps scrub ok 2024-09-16T08:10:17.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:10:17 smithi028 ceph-mon[23744]: pgmap v122: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-16T08:10:17.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:10:17 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-16T08:10:17.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:10:17 smithi028 ceph-mon[23744]: osd.5 [v2:172.21.15.144:6808/2187592584,v1:172.21.15.144:6809/2187592584] boot 2024-09-16T08:10:17.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:10:17 smithi028 ceph-mon[23744]: osdmap e39: 6 total, 6 up, 6 in 2024-09-16T08:10:17.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:10:17 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-16T08:10:18.445 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:10:18 smithi144 ceph-mon[25668]: osdmap e40: 6 total, 6 up, 6 in 2024-09-16T08:10:18.446 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:10:18 smithi028 ceph-mon[23744]: osdmap e40: 6 total, 6 up, 6 in 2024-09-16T08:10:18.449 INFO:teuthology.orchestra.run.smithi144.stdout: 2024-09-16T08:10:18.470 DEBUG:teuthology.orchestra.run.smithi144:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5622b2fe-7402-11ef-bceb-c7b262605968 -- ceph orch daemon add osd smithi144:vg_nvme/lv_2 2024-09-16T08:10:19.293 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:10:19 smithi144 ceph-mon[25668]: pgmap v125: 1 pgs: 1 remapped+peering; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-09-16T08:10:19.294 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:10:19 smithi144 ceph-mon[25668]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-09-16T08:10:19.294 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:10:19 smithi144 ceph-mon[25668]: osdmap e41: 6 total, 6 up, 6 in 2024-09-16T08:10:19.294 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:10:19 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' 2024-09-16T08:10:19.446 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:10:19 smithi028 ceph-mon[23744]: pgmap v125: 1 pgs: 1 remapped+peering; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-09-16T08:10:19.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:10:19 smithi028 ceph-mon[23744]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-09-16T08:10:19.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:10:19 smithi028 ceph-mon[23744]: osdmap e41: 6 total, 6 up, 6 in 2024-09-16T08:10:19.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:10:19 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' 2024-09-16T08:10:20.920 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:10:20 smithi144 ceph-mon[25668]: pgmap v127: 1 pgs: 1 remapped+peering; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-09-16T08:10:20.920 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:10:20 smithi144 ceph-mon[25668]: osdmap e42: 6 total, 6 up, 6 in 2024-09-16T08:10:20.920 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:10:20 smithi144 ceph-mon[25668]: Detected new or changed devices on smithi144 2024-09-16T08:10:20.921 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:10:20 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' 2024-09-16T08:10:20.921 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:10:20 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-16T08:10:20.921 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:10:20 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-16T08:10:20.921 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:10:20 smithi144 ceph-mon[25668]: Adjusting osd_memory_target on smithi144 to 7968M 2024-09-16T08:10:20.921 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:10:20 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' 2024-09-16T08:10:20.921 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:10:20 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' 2024-09-16T08:10:20.921 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:10:20 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-16T08:10:20.921 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:10:20 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T08:10:20.922 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:10:20 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-16T08:10:20.922 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:10:20 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' 2024-09-16T08:10:20.946 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:10:20 smithi028 ceph-mon[23744]: pgmap v127: 1 pgs: 1 remapped+peering; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-09-16T08:10:20.947 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:10:20 smithi028 ceph-mon[23744]: osdmap e42: 6 total, 6 up, 6 in 2024-09-16T08:10:20.947 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:10:20 smithi028 ceph-mon[23744]: Detected new or changed devices on smithi144 2024-09-16T08:10:20.947 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:10:20 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' 2024-09-16T08:10:20.947 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:10:20 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-16T08:10:20.947 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:10:20 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-16T08:10:20.947 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:10:20 smithi028 ceph-mon[23744]: Adjusting osd_memory_target on smithi144 to 7968M 2024-09-16T08:10:20.947 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:10:20 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' 2024-09-16T08:10:20.947 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:10:20 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' 2024-09-16T08:10:20.947 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:10:20 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-16T08:10:20.948 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:10:20 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T08:10:20.948 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:10:20 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-16T08:10:20.948 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:10:20 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' 2024-09-16T08:10:21.942 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:10:21 smithi144 ceph-mon[25668]: osdmap e43: 6 total, 6 up, 6 in 2024-09-16T08:10:21.943 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:10:21 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-16T08:10:21.943 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:10:21 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-16T08:10:21.943 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:10:21 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T08:10:21.946 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:10:21 smithi028 ceph-mon[23744]: osdmap e43: 6 total, 6 up, 6 in 2024-09-16T08:10:21.946 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:10:21 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-16T08:10:21.946 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:10:21 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-16T08:10:21.946 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:10:21 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T08:10:22.779 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:10:22 smithi144 ceph-mon[25668]: from='client.24193 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi144:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T08:10:22.779 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:10:22 smithi144 ceph-mon[25668]: pgmap v130: 1 pgs: 1 remapped+peering; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-16T08:10:22.946 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:10:22 smithi028 ceph-mon[23744]: from='client.24193 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi144:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T08:10:22.946 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:10:22 smithi028 ceph-mon[23744]: pgmap v130: 1 pgs: 1 remapped+peering; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-16T08:10:24.081 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:10:23 smithi144 ceph-mon[25668]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-09-16T08:10:24.082 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:10:23 smithi144 ceph-mon[25668]: Cluster is now healthy 2024-09-16T08:10:24.196 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:10:23 smithi028 ceph-mon[23744]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-09-16T08:10:24.196 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:10:23 smithi028 ceph-mon[23744]: Cluster is now healthy 2024-09-16T08:10:25.081 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:10:24 smithi144 ceph-mon[25668]: pgmap v131: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-09-16T08:10:25.081 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:10:24 smithi144 ceph-mon[25668]: from='client.? 172.21.15.144:0/1014044034' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "f19f02ff-b250-46a2-8f33-183bde556e09"}]: dispatch 2024-09-16T08:10:25.081 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:10:24 smithi144 ceph-mon[25668]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "f19f02ff-b250-46a2-8f33-183bde556e09"}]: dispatch 2024-09-16T08:10:25.082 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:10:24 smithi144 ceph-mon[25668]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "f19f02ff-b250-46a2-8f33-183bde556e09"}]': finished 2024-09-16T08:10:25.082 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:10:24 smithi144 ceph-mon[25668]: osdmap e44: 7 total, 6 up, 7 in 2024-09-16T08:10:25.082 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:10:24 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-16T08:10:25.082 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:10:24 smithi144 ceph-mon[25668]: from='client.? 172.21.15.144:0/2760357621' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-16T08:10:25.197 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:10:24 smithi028 ceph-mon[23744]: pgmap v131: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-09-16T08:10:25.197 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:10:24 smithi028 ceph-mon[23744]: from='client.? 172.21.15.144:0/1014044034' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "f19f02ff-b250-46a2-8f33-183bde556e09"}]: dispatch 2024-09-16T08:10:25.197 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:10:24 smithi028 ceph-mon[23744]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "f19f02ff-b250-46a2-8f33-183bde556e09"}]: dispatch 2024-09-16T08:10:25.197 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:10:24 smithi028 ceph-mon[23744]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "f19f02ff-b250-46a2-8f33-183bde556e09"}]': finished 2024-09-16T08:10:25.197 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:10:24 smithi028 ceph-mon[23744]: osdmap e44: 7 total, 6 up, 7 in 2024-09-16T08:10:25.197 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:10:24 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-16T08:10:25.197 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:10:24 smithi028 ceph-mon[23744]: from='client.? 172.21.15.144:0/2760357621' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-16T08:10:27.196 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:10:27 smithi144 ceph-mon[25668]: pgmap v133: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-09-16T08:10:27.446 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:10:27 smithi028 ceph-mon[23744]: pgmap v133: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-09-16T08:10:29.221 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:10:29 smithi144 ceph-mon[25668]: pgmap v134: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-09-16T08:10:29.446 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:10:29 smithi028 ceph-mon[23744]: pgmap v134: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-09-16T08:10:30.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:10:30 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-09-16T08:10:30.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:10:30 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T08:10:30.348 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:10:30 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-09-16T08:10:30.348 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:10:30 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T08:10:31.290 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:10:31 smithi144 ceph-mon[25668]: pgmap v135: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-09-16T08:10:31.290 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:10:31 smithi144 ceph-mon[25668]: Deploying daemon osd.6 on smithi144 2024-09-16T08:10:31.446 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:10:31 smithi028 ceph-mon[23744]: pgmap v135: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-09-16T08:10:31.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:10:31 smithi028 ceph-mon[23744]: Deploying daemon osd.6 on smithi144 2024-09-16T08:10:33.181 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:10:33 smithi144 ceph-mon[25668]: pgmap v136: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-09-16T08:10:33.446 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:10:33 smithi028 ceph-mon[23744]: pgmap v136: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-09-16T08:10:34.694 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:10:34 smithi144 ceph-mon[25668]: pgmap v137: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-16T08:10:34.695 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:10:34 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' 2024-09-16T08:10:34.946 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:10:34 smithi028 ceph-mon[23744]: pgmap v137: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-16T08:10:34.947 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:10:34 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' 2024-09-16T08:10:36.196 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:10:35 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' 2024-09-16T08:10:36.197 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:10:35 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-16T08:10:36.197 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:10:35 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T08:10:36.197 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:10:35 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-16T08:10:36.229 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:10:35 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' 2024-09-16T08:10:36.229 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:10:35 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-16T08:10:36.229 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:10:35 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T08:10:36.229 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:10:35 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-16T08:10:37.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:10:37 smithi144 ceph-mon[25668]: pgmap v138: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-16T08:10:37.446 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:10:37 smithi028 ceph-mon[23744]: pgmap v138: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-16T08:10:38.158 INFO:teuthology.orchestra.run.smithi144.stdout:Created osd(s) 6 on host 'smithi144' 2024-09-16T08:10:38.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:10:38 smithi144 ceph-mon[25668]: from='osd.6 [v2:172.21.15.144:6816/3907677683,v1:172.21.15.144:6817/3907677683]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-09-16T08:10:38.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:10:38 smithi144 ceph-mon[25668]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-09-16T08:10:38.446 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:10:38 smithi028 ceph-mon[23744]: from='osd.6 [v2:172.21.15.144:6816/3907677683,v1:172.21.15.144:6817/3907677683]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-09-16T08:10:38.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:10:38 smithi028 ceph-mon[23744]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-09-16T08:10:39.171 DEBUG:teuthology.orchestra.run.smithi144:osd.6> sudo journalctl -f -n 0 -u ceph-5622b2fe-7402-11ef-bceb-c7b262605968@osd.6.service 2024-09-16T08:10:39.174 INFO:tasks.cephadm:Deploying osd.7 on smithi144 with /dev/vg_nvme/lv_1... 2024-09-16T08:10:39.174 DEBUG:teuthology.orchestra.run.smithi144:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5622b2fe-7402-11ef-bceb-c7b262605968 -- lvm zap /dev/vg_nvme/lv_1 2024-09-16T08:10:39.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:10:39 smithi144 ceph-mon[25668]: pgmap v139: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-16T08:10:39.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:10:39 smithi144 ceph-mon[25668]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-09-16T08:10:39.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:10:39 smithi144 ceph-mon[25668]: osdmap e45: 7 total, 6 up, 7 in 2024-09-16T08:10:39.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:10:39 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-16T08:10:39.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:10:39 smithi144 ceph-mon[25668]: from='osd.6 [v2:172.21.15.144:6816/3907677683,v1:172.21.15.144:6817/3907677683]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi144", "root=default"]}]: dispatch 2024-09-16T08:10:39.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:10:39 smithi144 ceph-mon[25668]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi144", "root=default"]}]: dispatch 2024-09-16T08:10:39.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:10:39 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' 2024-09-16T08:10:39.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:10:39 smithi028 ceph-mon[23744]: pgmap v139: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-16T08:10:39.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:10:39 smithi028 ceph-mon[23744]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-09-16T08:10:39.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:10:39 smithi028 ceph-mon[23744]: osdmap e45: 7 total, 6 up, 7 in 2024-09-16T08:10:39.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:10:39 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-16T08:10:39.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:10:39 smithi028 ceph-mon[23744]: from='osd.6 [v2:172.21.15.144:6816/3907677683,v1:172.21.15.144:6817/3907677683]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi144", "root=default"]}]: dispatch 2024-09-16T08:10:39.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:10:39 smithi028 ceph-mon[23744]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi144", "root=default"]}]: dispatch 2024-09-16T08:10:39.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:10:39 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' 2024-09-16T08:10:40.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:10:40 smithi144 ceph-mon[25668]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi144", "root=default"]}]': finished 2024-09-16T08:10:40.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:10:40 smithi144 ceph-mon[25668]: osdmap e46: 7 total, 6 up, 7 in 2024-09-16T08:10:40.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:10:40 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-16T08:10:40.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:10:40 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-16T08:10:40.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:10:40 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' 2024-09-16T08:10:40.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:10:40 smithi144 ceph-mon[25668]: osd.6 [v2:172.21.15.144:6816/3907677683,v1:172.21.15.144:6817/3907677683] boot 2024-09-16T08:10:40.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:10:40 smithi144 ceph-mon[25668]: osdmap e47: 7 total, 7 up, 7 in 2024-09-16T08:10:40.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:10:40 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-16T08:10:40.446 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:10:40 smithi028 ceph-mon[23744]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi144", "root=default"]}]': finished 2024-09-16T08:10:40.446 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:10:40 smithi028 ceph-mon[23744]: osdmap e46: 7 total, 6 up, 7 in 2024-09-16T08:10:40.446 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:10:40 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-16T08:10:40.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:10:40 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-16T08:10:40.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:10:40 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' 2024-09-16T08:10:40.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:10:40 smithi028 ceph-mon[23744]: osd.6 [v2:172.21.15.144:6816/3907677683,v1:172.21.15.144:6817/3907677683] boot 2024-09-16T08:10:40.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:10:40 smithi028 ceph-mon[23744]: osdmap e47: 7 total, 7 up, 7 in 2024-09-16T08:10:40.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:10:40 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-16T08:10:41.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:10:41 smithi144 ceph-mon[25668]: purged_snaps scrub starts 2024-09-16T08:10:41.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:10:41 smithi144 ceph-mon[25668]: purged_snaps scrub ok 2024-09-16T08:10:41.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:10:41 smithi144 ceph-mon[25668]: pgmap v142: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-16T08:10:41.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:10:41 smithi144 ceph-mon[25668]: osdmap e48: 7 total, 7 up, 7 in 2024-09-16T08:10:41.446 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:10:41 smithi028 ceph-mon[23744]: purged_snaps scrub starts 2024-09-16T08:10:41.446 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:10:41 smithi028 ceph-mon[23744]: purged_snaps scrub ok 2024-09-16T08:10:41.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:10:41 smithi028 ceph-mon[23744]: pgmap v142: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-16T08:10:41.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:10:41 smithi028 ceph-mon[23744]: osdmap e48: 7 total, 7 up, 7 in 2024-09-16T08:10:42.660 INFO:teuthology.orchestra.run.smithi144.stdout: 2024-09-16T08:10:42.682 DEBUG:teuthology.orchestra.run.smithi144:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5622b2fe-7402-11ef-bceb-c7b262605968 -- ceph orch daemon add osd smithi144:vg_nvme/lv_1 2024-09-16T08:10:43.309 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:10:43 smithi144 ceph-mon[25668]: pgmap v145: 1 pgs: 1 active+clean; 577 KiB data, 40 MiB used, 626 GiB / 626 GiB avail 2024-09-16T08:10:43.309 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:10:43 smithi144 ceph-mon[25668]: osdmap e49: 7 total, 7 up, 7 in 2024-09-16T08:10:43.446 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:10:43 smithi028 ceph-mon[23744]: pgmap v145: 1 pgs: 1 active+clean; 577 KiB data, 40 MiB used, 626 GiB / 626 GiB avail 2024-09-16T08:10:43.446 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:10:43 smithi028 ceph-mon[23744]: osdmap e49: 7 total, 7 up, 7 in 2024-09-16T08:10:44.347 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:10:44 smithi144 ceph-mon[25668]: osdmap e50: 7 total, 7 up, 7 in 2024-09-16T08:10:44.446 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:10:44 smithi028 ceph-mon[23744]: osdmap e50: 7 total, 7 up, 7 in 2024-09-16T08:10:45.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:10:45 smithi144 ceph-mon[25668]: pgmap v148: 1 pgs: 1 activating+undersized+degraded+remapped; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 2/6 objects degraded (33.333%) 2024-09-16T08:10:45.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:10:45 smithi144 ceph-mon[25668]: Health check failed: Degraded data redundancy: 2/6 objects degraded (33.333%), 1 pg degraded (PG_DEGRADED) 2024-09-16T08:10:45.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:10:45 smithi144 ceph-mon[25668]: osdmap e51: 7 total, 7 up, 7 in 2024-09-16T08:10:45.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:10:45 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-16T08:10:45.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:10:45 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-16T08:10:45.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:10:45 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T08:10:45.446 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:10:45 smithi028 ceph-mon[23744]: pgmap v148: 1 pgs: 1 activating+undersized+degraded+remapped; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 2/6 objects degraded (33.333%) 2024-09-16T08:10:45.446 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:10:45 smithi028 ceph-mon[23744]: Health check failed: Degraded data redundancy: 2/6 objects degraded (33.333%), 1 pg degraded (PG_DEGRADED) 2024-09-16T08:10:45.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:10:45 smithi028 ceph-mon[23744]: osdmap e51: 7 total, 7 up, 7 in 2024-09-16T08:10:45.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:10:45 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-16T08:10:45.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:10:45 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-16T08:10:45.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:10:45 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T08:10:46.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:10:46 smithi144 ceph-mon[25668]: from='client.24211 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi144:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T08:10:46.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:10:46 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' 2024-09-16T08:10:46.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:10:46 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-16T08:10:46.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:10:46 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-16T08:10:46.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:10:46 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-16T08:10:46.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:10:46 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' 2024-09-16T08:10:46.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:10:46 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' 2024-09-16T08:10:46.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:10:46 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-16T08:10:46.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:10:46 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T08:10:46.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:10:46 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-16T08:10:46.333 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:10:46 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' 2024-09-16T08:10:46.446 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:10:46 smithi028 ceph-mon[23744]: from='client.24211 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi144:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T08:10:46.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:10:46 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' 2024-09-16T08:10:46.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:10:46 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-16T08:10:46.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:10:46 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-16T08:10:46.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:10:46 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-16T08:10:46.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:10:46 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' 2024-09-16T08:10:46.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:10:46 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' 2024-09-16T08:10:46.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:10:46 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-16T08:10:46.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:10:46 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T08:10:46.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:10:46 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-16T08:10:46.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:10:46 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' 2024-09-16T08:10:47.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:10:47 smithi144 ceph-mon[25668]: Detected new or changed devices on smithi144 2024-09-16T08:10:47.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:10:47 smithi144 ceph-mon[25668]: Adjusting osd_memory_target on smithi144 to 5312M 2024-09-16T08:10:47.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:10:47 smithi144 ceph-mon[25668]: pgmap v150: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 113 KiB/s, 0 objects/s recovering 2024-09-16T08:10:47.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:10:47 smithi144 ceph-mon[25668]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/6 objects degraded (33.333%), 1 pg degraded) 2024-09-16T08:10:47.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:10:47 smithi144 ceph-mon[25668]: Cluster is now healthy 2024-09-16T08:10:47.446 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:10:47 smithi028 ceph-mon[23744]: Detected new or changed devices on smithi144 2024-09-16T08:10:47.446 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:10:47 smithi028 ceph-mon[23744]: Adjusting osd_memory_target on smithi144 to 5312M 2024-09-16T08:10:47.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:10:47 smithi028 ceph-mon[23744]: pgmap v150: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 113 KiB/s, 0 objects/s recovering 2024-09-16T08:10:47.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:10:47 smithi028 ceph-mon[23744]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/6 objects degraded (33.333%), 1 pg degraded) 2024-09-16T08:10:47.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:10:47 smithi028 ceph-mon[23744]: Cluster is now healthy 2024-09-16T08:10:48.446 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:10:48 smithi028 ceph-mon[23744]: from='client.? 172.21.15.144:0/903085123' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "812947ad-33a8-440b-882e-aacc18adc2ef"}]: dispatch 2024-09-16T08:10:48.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:10:48 smithi028 ceph-mon[23744]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "812947ad-33a8-440b-882e-aacc18adc2ef"}]: dispatch 2024-09-16T08:10:48.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:10:48 smithi028 ceph-mon[23744]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "812947ad-33a8-440b-882e-aacc18adc2ef"}]': finished 2024-09-16T08:10:48.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:10:48 smithi028 ceph-mon[23744]: osdmap e52: 8 total, 7 up, 8 in 2024-09-16T08:10:48.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:10:48 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-16T08:10:48.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:10:48 smithi144 ceph-mon[25668]: from='client.? 172.21.15.144:0/903085123' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "812947ad-33a8-440b-882e-aacc18adc2ef"}]: dispatch 2024-09-16T08:10:48.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:10:48 smithi144 ceph-mon[25668]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "812947ad-33a8-440b-882e-aacc18adc2ef"}]: dispatch 2024-09-16T08:10:48.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:10:48 smithi144 ceph-mon[25668]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "812947ad-33a8-440b-882e-aacc18adc2ef"}]': finished 2024-09-16T08:10:48.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:10:48 smithi144 ceph-mon[25668]: osdmap e52: 8 total, 7 up, 8 in 2024-09-16T08:10:48.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:10:48 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-16T08:10:49.446 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:10:49 smithi028 ceph-mon[23744]: pgmap v152: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 93 KiB/s, 0 objects/s recovering 2024-09-16T08:10:49.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:10:49 smithi028 ceph-mon[23744]: from='client.? 172.21.15.144:0/193520661' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-16T08:10:49.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:10:49 smithi144 ceph-mon[25668]: pgmap v152: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 93 KiB/s, 0 objects/s recovering 2024-09-16T08:10:49.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:10:49 smithi144 ceph-mon[25668]: from='client.? 172.21.15.144:0/193520661' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-16T08:10:51.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:10:51 smithi144 ceph-mon[25668]: pgmap v153: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 79 KiB/s, 0 objects/s recovering 2024-09-16T08:10:51.446 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:10:51 smithi028 ceph-mon[23744]: pgmap v153: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 79 KiB/s, 0 objects/s recovering 2024-09-16T08:10:53.442 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:10:53 smithi144 ceph-mon[25668]: pgmap v154: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-09-16T08:10:53.446 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:10:53 smithi028 ceph-mon[23744]: pgmap v154: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-09-16T08:10:54.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:10:54 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-09-16T08:10:54.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:10:54 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T08:10:54.446 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:10:54 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-09-16T08:10:54.446 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:10:54 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T08:10:55.382 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:10:55 smithi144 ceph-mon[25668]: pgmap v155: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 54 KiB/s, 0 objects/s recovering 2024-09-16T08:10:55.383 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:10:55 smithi144 ceph-mon[25668]: Deploying daemon osd.7 on smithi144 2024-09-16T08:10:55.446 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:10:55 smithi028 ceph-mon[23744]: pgmap v155: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 54 KiB/s, 0 objects/s recovering 2024-09-16T08:10:55.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:10:55 smithi028 ceph-mon[23744]: Deploying daemon osd.7 on smithi144 2024-09-16T08:10:57.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:10:57 smithi144 ceph-mon[25668]: pgmap v156: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-09-16T08:10:57.446 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:10:57 smithi028 ceph-mon[23744]: pgmap v156: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-09-16T08:10:59.192 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:10:59 smithi144 ceph-mon[25668]: pgmap v157: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-09-16T08:10:59.235 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:10:59 smithi028 ceph-mon[23744]: pgmap v157: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-09-16T08:11:00.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:11:00 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' 2024-09-16T08:11:00.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:11:00 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-16T08:11:00.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:11:00 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T08:11:00.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:11:00 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-16T08:11:00.558 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:11:00 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' 2024-09-16T08:11:00.558 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:11:00 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-16T08:11:00.558 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:11:00 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T08:11:00.558 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:11:00 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-16T08:11:01.446 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:11:01 smithi028 ceph-mon[23744]: pgmap v158: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-09-16T08:11:01.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:11:01 smithi144 ceph-mon[25668]: pgmap v158: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-09-16T08:11:02.675 INFO:teuthology.orchestra.run.smithi144.stdout:Created osd(s) 7 on host 'smithi144' 2024-09-16T08:11:03.291 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:11:03 smithi144 ceph-mon[25668]: pgmap v159: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-09-16T08:11:03.291 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:11:03 smithi144 ceph-mon[25668]: from='osd.7 [v2:172.21.15.144:6824/1629206793,v1:172.21.15.144:6825/1629206793]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-09-16T08:11:03.291 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:11:03 smithi144 ceph-mon[25668]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-09-16T08:11:03.291 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:11:03 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' 2024-09-16T08:11:03.312 DEBUG:teuthology.orchestra.run.smithi144:osd.7> sudo journalctl -f -n 0 -u ceph-5622b2fe-7402-11ef-bceb-c7b262605968@osd.7.service 2024-09-16T08:11:03.315 INFO:tasks.cephadm:Waiting for 8 OSDs to come up... 2024-09-16T08:11:03.315 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5622b2fe-7402-11ef-bceb-c7b262605968 -- ceph osd stat -f json 2024-09-16T08:11:03.446 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:11:03 smithi028 ceph-mon[23744]: pgmap v159: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-09-16T08:11:03.446 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:11:03 smithi028 ceph-mon[23744]: from='osd.7 [v2:172.21.15.144:6824/1629206793,v1:172.21.15.144:6825/1629206793]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-09-16T08:11:03.446 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:11:03 smithi028 ceph-mon[23744]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-09-16T08:11:03.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:11:03 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' 2024-09-16T08:11:04.446 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:11:04 smithi028 ceph-mon[23744]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-09-16T08:11:04.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:11:04 smithi028 ceph-mon[23744]: osdmap e53: 8 total, 7 up, 8 in 2024-09-16T08:11:04.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:11:04 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-16T08:11:04.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:11:04 smithi028 ceph-mon[23744]: from='osd.7 [v2:172.21.15.144:6824/1629206793,v1:172.21.15.144:6825/1629206793]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi144", "root=default"]}]: dispatch 2024-09-16T08:11:04.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:11:04 smithi028 ceph-mon[23744]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi144", "root=default"]}]: dispatch 2024-09-16T08:11:04.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:11:04 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi028.azvyzy/mirror_snapshot_schedule"}]: dispatch 2024-09-16T08:11:04.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:11:04 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi028.azvyzy/trash_purge_schedule"}]: dispatch 2024-09-16T08:11:04.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:11:04 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' 2024-09-16T08:11:04.458 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:11:04 smithi144 ceph-mon[25668]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-09-16T08:11:04.458 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:11:04 smithi144 ceph-mon[25668]: osdmap e53: 8 total, 7 up, 8 in 2024-09-16T08:11:04.458 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:11:04 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-16T08:11:04.459 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:11:04 smithi144 ceph-mon[25668]: from='osd.7 [v2:172.21.15.144:6824/1629206793,v1:172.21.15.144:6825/1629206793]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi144", "root=default"]}]: dispatch 2024-09-16T08:11:04.459 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:11:04 smithi144 ceph-mon[25668]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi144", "root=default"]}]: dispatch 2024-09-16T08:11:04.459 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:11:04 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi028.azvyzy/mirror_snapshot_schedule"}]: dispatch 2024-09-16T08:11:04.459 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:11:04 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi028.azvyzy/trash_purge_schedule"}]: dispatch 2024-09-16T08:11:04.459 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:11:04 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' 2024-09-16T08:11:04.459 INFO:journalctl@ceph.osd.7.smithi144.stdout:Sep 16 08:11:04 smithi144 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-7[38675]: 2024-09-16T08:11:04.188+0000 7f92aaf7b700 -1 osd.7 0 waiting for initial osdmap 2024-09-16T08:11:04.459 INFO:journalctl@ceph.osd.7.smithi144.stdout:Sep 16 08:11:04 smithi144 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-7[38675]: 2024-09-16T08:11:04.195+0000 7f92a6113700 -1 osd.7 54 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-16T08:11:05.197 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:11:05 smithi028 ceph-mon[23744]: pgmap v161: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-09-16T08:11:05.197 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:11:05 smithi028 ceph-mon[23744]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi144", "root=default"]}]': finished 2024-09-16T08:11:05.197 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:11:05 smithi028 ceph-mon[23744]: osdmap e54: 8 total, 7 up, 8 in 2024-09-16T08:11:05.197 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:11:05 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-16T08:11:05.197 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:11:05 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-16T08:11:05.197 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:11:05 smithi028 ceph-mon[23744]: osd.7 [v2:172.21.15.144:6824/1629206793,v1:172.21.15.144:6825/1629206793] boot 2024-09-16T08:11:05.197 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:11:05 smithi028 ceph-mon[23744]: osdmap e55: 8 total, 8 up, 8 in 2024-09-16T08:11:05.197 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:11:05 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-16T08:11:05.509 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:11:05 smithi144 ceph-mon[25668]: pgmap v161: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-09-16T08:11:05.509 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:11:05 smithi144 ceph-mon[25668]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi144", "root=default"]}]': finished 2024-09-16T08:11:05.509 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:11:05 smithi144 ceph-mon[25668]: osdmap e54: 8 total, 7 up, 8 in 2024-09-16T08:11:05.509 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:11:05 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-16T08:11:05.509 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:11:05 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-16T08:11:05.510 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:11:05 smithi144 ceph-mon[25668]: osd.7 [v2:172.21.15.144:6824/1629206793,v1:172.21.15.144:6825/1629206793] boot 2024-09-16T08:11:05.510 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:11:05 smithi144 ceph-mon[25668]: osdmap e55: 8 total, 8 up, 8 in 2024-09-16T08:11:05.510 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:11:05 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-16T08:11:05.611 INFO:teuthology.orchestra.run.smithi028.stdout: 2024-09-16T08:11:06.373 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:11:06 smithi028 ceph-mon[23744]: purged_snaps scrub starts 2024-09-16T08:11:06.373 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:11:06 smithi028 ceph-mon[23744]: purged_snaps scrub ok 2024-09-16T08:11:06.373 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:11:06 smithi028 ceph-mon[23744]: from='client.? 172.21.15.28:0/763708689' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-09-16T08:11:06.373 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:11:06 smithi028 ceph-mon[23744]: osdmap e56: 8 total, 8 up, 8 in 2024-09-16T08:11:06.396 INFO:teuthology.orchestra.run.smithi028.stdout:{"epoch":55,"num_osds":8,"num_up_osds":8,"osd_up_since":1726474265,"num_in_osds":8,"osd_in_since":1726474247,"num_remapped_pgs":1} 2024-09-16T08:11:06.397 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 5622b2fe-7402-11ef-bceb-c7b262605968 -- ceph osd dump --format=json 2024-09-16T08:11:06.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:11:06 smithi144 ceph-mon[25668]: purged_snaps scrub starts 2024-09-16T08:11:06.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:11:06 smithi144 ceph-mon[25668]: purged_snaps scrub ok 2024-09-16T08:11:06.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:11:06 smithi144 ceph-mon[25668]: from='client.? 172.21.15.28:0/763708689' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-09-16T08:11:06.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:11:06 smithi144 ceph-mon[25668]: osdmap e56: 8 total, 8 up, 8 in 2024-09-16T08:11:06.616 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/5622b2fe-7402-11ef-bceb-c7b262605968/mon.smithi028/config 2024-09-16T08:11:07.325 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:11:07 smithi144 ceph-mon[25668]: pgmap v164: 1 pgs: 1 active+clean; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2024-09-16T08:11:07.325 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:11:07 smithi144 ceph-mon[25668]: osdmap e57: 8 total, 8 up, 8 in 2024-09-16T08:11:07.449 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:11:07 smithi028 ceph-mon[23744]: pgmap v164: 1 pgs: 1 active+clean; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2024-09-16T08:11:07.449 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:11:07 smithi028 ceph-mon[23744]: osdmap e57: 8 total, 8 up, 8 in 2024-09-16T08:11:08.331 INFO:teuthology.orchestra.run.smithi028.stdout: 2024-09-16T08:11:08.331 INFO:teuthology.orchestra.run.smithi028.stdout:{"epoch":58,"fsid":"5622b2fe-7402-11ef-bceb-c7b262605968","created":"2024-09-16T08:05:20.224194+0000","modified":"2024-09-16T08:11:08.074615+0000","last_up_change":"2024-09-16T08:11:05.067238+0000","last_in_change":"2024-09-16T08:10:47.598994+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-16T08:09:06.094185+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}}],"osds":[{"osd":0,"uuid":"66f613ac-6dba-4ca1-ab30-8b95e06260c0","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6802","nonce":230701446},{"type":"v1","addr":"172.21.15.28:6803","nonce":230701446}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6804","nonce":230701446},{"type":"v1","addr":"172.21.15.28:6805","nonce":230701446}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6808","nonce":230701446},{"type":"v1","addr":"172.21.15.28:6809","nonce":230701446}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6806","nonce":230701446},{"type":"v1","addr":"172.21.15.28:6807","nonce":230701446}]},"public_addr":"172.21.15.28:6803/230701446","cluster_addr":"172.21.15.28:6805/230701446","heartbeat_back_addr":"172.21.15.28:6809/230701446","heartbeat_front_addr":"172.21.15.28:6807/230701446","state":["exists","up"]},{"osd":1,"uuid":"524eec0a-00eb-403b-a964-ae56bd9eb6f5","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":34,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6810","nonce":3124900380},{"type":"v1","addr":"172.21.15.28:6811","nonce":3124900380}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6812","nonce":3124900380},{"type":"v1","addr":"172.21.15.28:6813","nonce":3124900380}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6816","nonce":3124900380},{"type":"v1","addr":"172.21.15.28:6817","nonce":3124900380}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6814","nonce":3124900380},{"type":"v1","addr":"172.21.15.28:6815","nonce":3124900380}]},"public_addr":"172.21.15.28:6811/3124900380","cluster_addr":"172.21.15.28:6813/3124900380","heartbeat_back_addr":"172.21.15.28:6817/3124900380","heartbeat_front_addr":"172.21.15.28:6815/3124900380","state":["exists","up"]},{"osd":2,"uuid":"cabe96b6-9d98-487a-8a56-6ef052c609f1","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":19,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6818","nonce":3138093499},{"type":"v1","addr":"172.21.15.28:6819","nonce":3138093499}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6820","nonce":3138093499},{"type":"v1","addr":"172.21.15.28:6821","nonce":3138093499}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6824","nonce":3138093499},{"type":"v1","addr":"172.21.15.28:6825","nonce":3138093499}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6822","nonce":3138093499},{"type":"v1","addr":"172.21.15.28:6823","nonce":3138093499}]},"public_addr":"172.21.15.28:6819/3138093499","cluster_addr":"172.21.15.28:6821/3138093499","heartbeat_back_addr":"172.21.15.28:6825/3138093499","heartbeat_front_addr":"172.21.15.28:6823/3138093499","state":["exists","up"]},{"osd":3,"uuid":"2400d76b-78e4-43dd-becc-d1f80427f1df","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6826","nonce":3665954147},{"type":"v1","addr":"172.21.15.28:6827","nonce":3665954147}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6828","nonce":3665954147},{"type":"v1","addr":"172.21.15.28:6829","nonce":3665954147}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6832","nonce":3665954147},{"type":"v1","addr":"172.21.15.28:6833","nonce":3665954147}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6830","nonce":3665954147},{"type":"v1","addr":"172.21.15.28:6831","nonce":3665954147}]},"public_addr":"172.21.15.28:6827/3665954147","cluster_addr":"172.21.15.28:6829/3665954147","heartbeat_back_addr":"172.21.15.28:6833/3665954147","heartbeat_front_addr":"172.21.15.28:6831/3665954147","state":["exists","up"]},{"osd":4,"uuid":"a929d7e2-fac0-47fe-9919-a4329c008de2","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6800","nonce":1792696254},{"type":"v1","addr":"172.21.15.144:6801","nonce":1792696254}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6802","nonce":1792696254},{"type":"v1","addr":"172.21.15.144:6803","nonce":1792696254}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6806","nonce":1792696254},{"type":"v1","addr":"172.21.15.144:6807","nonce":1792696254}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6804","nonce":1792696254},{"type":"v1","addr":"172.21.15.144:6805","nonce":1792696254}]},"public_addr":"172.21.15.144:6801/1792696254","cluster_addr":"172.21.15.144:6803/1792696254","heartbeat_back_addr":"172.21.15.144:6807/1792696254","heartbeat_front_addr":"172.21.15.144:6805/1792696254","state":["exists","up"]},{"osd":5,"uuid":"3f1ee7e1-e3ee-4351-8492-3573e3cb0fea","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":42,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6808","nonce":2187592584},{"type":"v1","addr":"172.21.15.144:6809","nonce":2187592584}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6810","nonce":2187592584},{"type":"v1","addr":"172.21.15.144:6811","nonce":2187592584}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6814","nonce":2187592584},{"type":"v1","addr":"172.21.15.144:6815","nonce":2187592584}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6812","nonce":2187592584},{"type":"v1","addr":"172.21.15.144:6813","nonce":2187592584}]},"public_addr":"172.21.15.144:6809/2187592584","cluster_addr":"172.21.15.144:6811/2187592584","heartbeat_back_addr":"172.21.15.144:6815/2187592584","heartbeat_front_addr":"172.21.15.144:6813/2187592584","state":["exists","up"]},{"osd":6,"uuid":"f19f02ff-b250-46a2-8f33-183bde556e09","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":50,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6816","nonce":3907677683},{"type":"v1","addr":"172.21.15.144:6817","nonce":3907677683}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6818","nonce":3907677683},{"type":"v1","addr":"172.21.15.144:6819","nonce":3907677683}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6822","nonce":3907677683},{"type":"v1","addr":"172.21.15.144:6823","nonce":3907677683}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6820","nonce":3907677683},{"type":"v1","addr":"172.21.15.144:6821","nonce":3907677683}]},"public_addr":"172.21.15.144:6817/3907677683","cluster_addr":"172.21.15.144:6819/3907677683","heartbeat_back_addr":"172.21.15.144:6823/3907677683","heartbeat_front_addr":"172.21.15.144:6821/3907677683","state":["exists","up"]},{"osd":7,"uuid":"812947ad-33a8-440b-882e-aacc18adc2ef","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6824","nonce":1629206793},{"type":"v1","addr":"172.21.15.144:6825","nonce":1629206793}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6826","nonce":1629206793},{"type":"v1","addr":"172.21.15.144:6827","nonce":1629206793}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6830","nonce":1629206793},{"type":"v1","addr":"172.21.15.144:6831","nonce":1629206793}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6828","nonce":1629206793},{"type":"v1","addr":"172.21.15.144:6829","nonce":1629206793}]},"public_addr":"172.21.15.144:6825/1629206793","cluster_addr":"172.21.15.144:6827/1629206793","heartbeat_back_addr":"172.21.15.144:6831/1629206793","heartbeat_front_addr":"172.21.15.144:6829/1629206793","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-16T08:08:09.177820+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-16T08:08:34.507057+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-16T08:09:02.169534+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-16T08:09:28.598100+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-16T08:09:50.082338+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-16T08:10:14.455953+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-16T08:10:38.954291+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-16T08:11:03.280241+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.28:0/3973422976":"2024-09-17T08:07:03.540703+0000","172.21.15.28:0/2703579451":"2024-09-17T08:07:03.540703+0000","172.21.15.28:6800/3593196739":"2024-09-17T08:07:03.540703+0000","172.21.15.28:0/1234187527":"2024-09-17T08:07:03.540703+0000","172.21.15.28:0/923827164":"2024-09-17T08:06:14.463773+0000","172.21.15.28:6801/24563670":"2024-09-17T08:06:14.463773+0000","172.21.15.28:6800/24563670":"2024-09-17T08:06:14.463773+0000","172.21.15.28:0/281327060":"2024-09-17T08:06:14.463773+0000","172.21.15.28:0/3672481329":"2024-09-17T08:05:45.461999+0000","172.21.15.28:0/2118149107":"2024-09-17T08:06:14.463773+0000","172.21.15.28:6801/282612086":"2024-09-17T08:05:45.461999+0000","172.21.15.28:0/3325094875":"2024-09-17T08:05:45.461999+0000","172.21.15.28:6801/3593196739":"2024-09-17T08:07:03.540703+0000","172.21.15.28:6800/282612086":"2024-09-17T08:05:45.461999+0000","172.21.15.28:0/3076918752":"2024-09-17T08:05:45.461999+0000"},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-16T08:11:08.697 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:11:08 smithi028 ceph-mon[23744]: Detected new or changed devices on smithi144 2024-09-16T08:11:08.697 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:11:08 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' 2024-09-16T08:11:08.697 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:11:08 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-16T08:11:08.697 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:11:08 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-16T08:11:08.697 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:11:08 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-16T08:11:08.698 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:11:08 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-16T08:11:08.698 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:11:08 smithi028 ceph-mon[23744]: Adjusting osd_memory_target on smithi144 to 3984M 2024-09-16T08:11:08.698 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:11:08 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' 2024-09-16T08:11:08.698 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:11:08 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' 2024-09-16T08:11:08.698 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:11:08 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-16T08:11:08.698 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:11:08 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T08:11:08.698 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:11:08 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-16T08:11:08.698 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:11:08 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' 2024-09-16T08:11:08.698 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:11:08 smithi028 ceph-mon[23744]: pgmap v167: 1 pgs: 1 active+recovering+undersized+remapped; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail; 1/6 objects misplaced (16.667%); 49 KiB/s, 0 objects/s recovering 2024-09-16T08:11:08.698 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:11:08 smithi028 ceph-mon[23744]: osdmap e58: 8 total, 8 up, 8 in 2024-09-16T08:11:08.698 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:11:08 smithi028 ceph-mon[23744]: from='client.? 172.21.15.28:0/2307651904' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-16T08:11:08.831 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:11:08 smithi144 ceph-mon[25668]: Detected new or changed devices on smithi144 2024-09-16T08:11:08.831 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:11:08 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' 2024-09-16T08:11:08.832 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:11:08 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-16T08:11:08.832 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:11:08 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-16T08:11:08.832 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:11:08 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-16T08:11:08.832 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:11:08 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-16T08:11:08.832 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:11:08 smithi144 ceph-mon[25668]: Adjusting osd_memory_target on smithi144 to 3984M 2024-09-16T08:11:08.832 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:11:08 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' 2024-09-16T08:11:08.832 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:11:08 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' 2024-09-16T08:11:08.832 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:11:08 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-16T08:11:08.832 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:11:08 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T08:11:08.832 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:11:08 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-16T08:11:08.832 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:11:08 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' 2024-09-16T08:11:08.832 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:11:08 smithi144 ceph-mon[25668]: pgmap v167: 1 pgs: 1 active+recovering+undersized+remapped; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail; 1/6 objects misplaced (16.667%); 49 KiB/s, 0 objects/s recovering 2024-09-16T08:11:08.832 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:11:08 smithi144 ceph-mon[25668]: osdmap e58: 8 total, 8 up, 8 in 2024-09-16T08:11:08.832 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:11:08 smithi144 ceph-mon[25668]: from='client.? 172.21.15.28:0/2307651904' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-16T08:11:09.054 INFO:tasks.cephadm.ceph_manager.ceph:[{'pool': 1, 'pool_name': '.mgr', 'create_time': '2024-09-16T08:09:06.094185+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '22', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}}] 2024-09-16T08:11:09.054 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 5622b2fe-7402-11ef-bceb-c7b262605968 -- ceph osd pool get .mgr pg_num 2024-09-16T08:11:09.276 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/5622b2fe-7402-11ef-bceb-c7b262605968/mon.smithi028/config 2024-09-16T08:11:10.196 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:11:10 smithi028 ceph-mon[23744]: osdmap e59: 8 total, 8 up, 8 in 2024-09-16T08:11:10.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:11:10 smithi144 ceph-mon[25668]: osdmap e59: 8 total, 8 up, 8 in 2024-09-16T08:11:10.991 INFO:teuthology.orchestra.run.smithi028.stdout:pg_num: 1 2024-09-16T08:11:11.446 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:11:11 smithi028 ceph-mon[23744]: pgmap v170: 1 pgs: 1 active+recovering+undersized+remapped; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail; 1/6 objects misplaced (16.667%); 49 KiB/s, 0 objects/s recovering 2024-09-16T08:11:11.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:11:11 smithi028 ceph-mon[23744]: from='client.? 172.21.15.28:0/2178871696' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-09-16T08:11:11.573 INFO:tasks.cephadm:Setting up client nodes... 2024-09-16T08:11:11.573 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5622b2fe-7402-11ef-bceb-c7b262605968 -- ceph auth get-or-create client.0 mon 'allow *' osd 'allow *' mds 'allow *' mgr 'allow *' 2024-09-16T08:11:11.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:11:11 smithi144 ceph-mon[25668]: pgmap v170: 1 pgs: 1 active+recovering+undersized+remapped; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail; 1/6 objects misplaced (16.667%); 49 KiB/s, 0 objects/s recovering 2024-09-16T08:11:11.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:11:11 smithi144 ceph-mon[25668]: from='client.? 172.21.15.28:0/2178871696' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-09-16T08:11:13.443 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:11:13 smithi144 ceph-mon[25668]: pgmap v171: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 93 KiB/s, 0 objects/s recovering 2024-09-16T08:11:13.446 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:11:13 smithi028 ceph-mon[23744]: pgmap v171: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 93 KiB/s, 0 objects/s recovering 2024-09-16T08:11:13.512 INFO:teuthology.orchestra.run.smithi028.stdout:[client.0] 2024-09-16T08:11:13.512 INFO:teuthology.orchestra.run.smithi028.stdout: key = AQAh6Odm+vNfHhAAXxIcUf8YyqoCqHWpMPz4pw== 2024-09-16T08:11:14.103 DEBUG:teuthology.orchestra.run.smithi028:> set -ex 2024-09-16T08:11:14.103 DEBUG:teuthology.orchestra.run.smithi028:> sudo dd of=/etc/ceph/ceph.client.0.keyring 2024-09-16T08:11:14.104 DEBUG:teuthology.orchestra.run.smithi028:> sudo chmod 0644 /etc/ceph/ceph.client.0.keyring 2024-09-16T08:11:14.128 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:11:14 smithi028 ceph-mon[23744]: from='client.? 172.21.15.28:0/3936024815' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-09-16T08:11:14.128 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:11:14 smithi028 ceph-mon[23744]: from='client.? 172.21.15.28:0/3936024815' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-09-16T08:11:14.144 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 5622b2fe-7402-11ef-bceb-c7b262605968 -- ceph config log 1 --format=json 2024-09-16T08:11:14.399 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/5622b2fe-7402-11ef-bceb-c7b262605968/mon.smithi028/config 2024-09-16T08:11:14.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:11:14 smithi144 ceph-mon[25668]: from='client.? 172.21.15.28:0/3936024815' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-09-16T08:11:14.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:11:14 smithi144 ceph-mon[25668]: from='client.? 172.21.15.28:0/3936024815' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-09-16T08:11:15.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:11:15 smithi028 ceph-mon[23744]: pgmap v172: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 79 KiB/s, 0 objects/s recovering 2024-09-16T08:11:15.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:11:15 smithi144 ceph-mon[25668]: pgmap v172: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 79 KiB/s, 0 objects/s recovering 2024-09-16T08:11:16.234 INFO:teuthology.orchestra.run.smithi028.stdout: 2024-09-16T08:11:16.941 INFO:teuthology.orchestra.run.smithi028.stdout:[{"version":20,"timestamp":"2024-09-16T08:11:07.360687+0000","name":"","changes":[{"name":"osd/host:smithi144/osd_memory_target","previous_value":"5570054963","new_value":"4177541222"}]}] 2024-09-16T08:11:16.941 INFO:tasks.ceph_manager:config epoch is 20 2024-09-16T08:11:16.942 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2024-09-16T08:11:16.942 INFO:tasks.cephadm.ceph_manager.ceph:waiting for mgr available 2024-09-16T08:11:16.942 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 5622b2fe-7402-11ef-bceb-c7b262605968 -- ceph mgr dump --format=json 2024-09-16T08:11:17.163 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/5622b2fe-7402-11ef-bceb-c7b262605968/mon.smithi028/config 2024-09-16T08:11:17.196 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:11:17 smithi028 ceph-mon[23744]: pgmap v173: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 40 KiB/s, 0 objects/s recovering 2024-09-16T08:11:17.196 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:11:17 smithi028 ceph-mon[23744]: from='client.? 172.21.15.28:0/2480789495' entity='client.admin' cmd=[{"prefix": "config log", "num": 1, "format": "json"}]: dispatch 2024-09-16T08:11:17.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:11:17 smithi144 ceph-mon[25668]: pgmap v173: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 40 KiB/s, 0 objects/s recovering 2024-09-16T08:11:17.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:11:17 smithi144 ceph-mon[25668]: from='client.? 172.21.15.28:0/2480789495' entity='client.admin' cmd=[{"prefix": "config log", "num": 1, "format": "json"}]: dispatch 2024-09-16T08:11:19.007 INFO:teuthology.orchestra.run.smithi028.stdout: 2024-09-16T08:11:19.325 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:11:19 smithi028 ceph-mon[23744]: pgmap v174: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 33 KiB/s, 0 objects/s recovering 2024-09-16T08:11:19.325 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:11:19 smithi028 ceph-mon[23744]: from='client.? 172.21.15.28:0/2618906098' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-09-16T08:11:19.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:11:19 smithi144 ceph-mon[25668]: pgmap v174: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 33 KiB/s, 0 objects/s recovering 2024-09-16T08:11:19.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:11:19 smithi144 ceph-mon[25668]: from='client.? 172.21.15.28:0/2618906098' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-09-16T08:11:19.643 INFO:teuthology.orchestra.run.smithi028.stdout:{"epoch":19,"active_gid":14202,"active_name":"smithi028.azvyzy","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6800","nonce":3147237525},{"type":"v1","addr":"172.21.15.28:6801","nonce":3147237525}]},"active_addr":"172.21.15.28:6801/3147237525","active_change":"2024-09-16T08:07:03.541055+0000","active_mgr_features":4540138303579357183,"available":true,"standbys":[{"gid":14220,"name":"smithi144.utzszj","mgr_features":4540138303579357183,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.23.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:8.3.5","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"docker.io/library/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"docker.io/arcts/keepalived","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.3.1","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.33.4","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.23.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:8.3.5","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"docker.io/library/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"docker.io/arcts/keepalived","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.3.1","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.33.4","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{"dashboard":"https://172.21.15.28:8443/","prometheus":"http://172.21.15.28:9283/"},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"last_failure_osd_epoch":5,"active_clients":[{"addrvec":[{"type":"v2","addr":"172.21.15.28:0","nonce":3229256436}]},{"addrvec":[{"type":"v2","addr":"172.21.15.28:0","nonce":3406998020}]},{"addrvec":[{"type":"v2","addr":"172.21.15.28:0","nonce":869254442}]},{"addrvec":[{"type":"v2","addr":"172.21.15.28:0","nonce":2095930858}]}]}} 2024-09-16T08:11:19.650 INFO:tasks.cephadm.ceph_manager.ceph:mgr available! 2024-09-16T08:11:19.650 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2024-09-16T08:11:19.650 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 5622b2fe-7402-11ef-bceb-c7b262605968 -- ceph osd dump --format=json 2024-09-16T08:11:19.873 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/5622b2fe-7402-11ef-bceb-c7b262605968/mon.smithi028/config 2024-09-16T08:11:21.327 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:11:21 smithi028 ceph-mon[23744]: pgmap v175: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 30 KiB/s, 0 objects/s recovering 2024-09-16T08:11:21.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:11:21 smithi144 ceph-mon[25668]: pgmap v175: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 30 KiB/s, 0 objects/s recovering 2024-09-16T08:11:21.730 INFO:teuthology.orchestra.run.smithi028.stdout: 2024-09-16T08:11:21.731 INFO:teuthology.orchestra.run.smithi028.stdout:{"epoch":59,"fsid":"5622b2fe-7402-11ef-bceb-c7b262605968","created":"2024-09-16T08:05:20.224194+0000","modified":"2024-09-16T08:11:09.075971+0000","last_up_change":"2024-09-16T08:11:05.067238+0000","last_in_change":"2024-09-16T08:10:47.598994+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-16T08:09:06.094185+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}}],"osds":[{"osd":0,"uuid":"66f613ac-6dba-4ca1-ab30-8b95e06260c0","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6802","nonce":230701446},{"type":"v1","addr":"172.21.15.28:6803","nonce":230701446}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6804","nonce":230701446},{"type":"v1","addr":"172.21.15.28:6805","nonce":230701446}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6808","nonce":230701446},{"type":"v1","addr":"172.21.15.28:6809","nonce":230701446}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6806","nonce":230701446},{"type":"v1","addr":"172.21.15.28:6807","nonce":230701446}]},"public_addr":"172.21.15.28:6803/230701446","cluster_addr":"172.21.15.28:6805/230701446","heartbeat_back_addr":"172.21.15.28:6809/230701446","heartbeat_front_addr":"172.21.15.28:6807/230701446","state":["exists","up"]},{"osd":1,"uuid":"524eec0a-00eb-403b-a964-ae56bd9eb6f5","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":34,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6810","nonce":3124900380},{"type":"v1","addr":"172.21.15.28:6811","nonce":3124900380}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6812","nonce":3124900380},{"type":"v1","addr":"172.21.15.28:6813","nonce":3124900380}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6816","nonce":3124900380},{"type":"v1","addr":"172.21.15.28:6817","nonce":3124900380}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6814","nonce":3124900380},{"type":"v1","addr":"172.21.15.28:6815","nonce":3124900380}]},"public_addr":"172.21.15.28:6811/3124900380","cluster_addr":"172.21.15.28:6813/3124900380","heartbeat_back_addr":"172.21.15.28:6817/3124900380","heartbeat_front_addr":"172.21.15.28:6815/3124900380","state":["exists","up"]},{"osd":2,"uuid":"cabe96b6-9d98-487a-8a56-6ef052c609f1","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":19,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6818","nonce":3138093499},{"type":"v1","addr":"172.21.15.28:6819","nonce":3138093499}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6820","nonce":3138093499},{"type":"v1","addr":"172.21.15.28:6821","nonce":3138093499}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6824","nonce":3138093499},{"type":"v1","addr":"172.21.15.28:6825","nonce":3138093499}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6822","nonce":3138093499},{"type":"v1","addr":"172.21.15.28:6823","nonce":3138093499}]},"public_addr":"172.21.15.28:6819/3138093499","cluster_addr":"172.21.15.28:6821/3138093499","heartbeat_back_addr":"172.21.15.28:6825/3138093499","heartbeat_front_addr":"172.21.15.28:6823/3138093499","state":["exists","up"]},{"osd":3,"uuid":"2400d76b-78e4-43dd-becc-d1f80427f1df","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6826","nonce":3665954147},{"type":"v1","addr":"172.21.15.28:6827","nonce":3665954147}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6828","nonce":3665954147},{"type":"v1","addr":"172.21.15.28:6829","nonce":3665954147}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6832","nonce":3665954147},{"type":"v1","addr":"172.21.15.28:6833","nonce":3665954147}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6830","nonce":3665954147},{"type":"v1","addr":"172.21.15.28:6831","nonce":3665954147}]},"public_addr":"172.21.15.28:6827/3665954147","cluster_addr":"172.21.15.28:6829/3665954147","heartbeat_back_addr":"172.21.15.28:6833/3665954147","heartbeat_front_addr":"172.21.15.28:6831/3665954147","state":["exists","up"]},{"osd":4,"uuid":"a929d7e2-fac0-47fe-9919-a4329c008de2","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6800","nonce":1792696254},{"type":"v1","addr":"172.21.15.144:6801","nonce":1792696254}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6802","nonce":1792696254},{"type":"v1","addr":"172.21.15.144:6803","nonce":1792696254}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6806","nonce":1792696254},{"type":"v1","addr":"172.21.15.144:6807","nonce":1792696254}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6804","nonce":1792696254},{"type":"v1","addr":"172.21.15.144:6805","nonce":1792696254}]},"public_addr":"172.21.15.144:6801/1792696254","cluster_addr":"172.21.15.144:6803/1792696254","heartbeat_back_addr":"172.21.15.144:6807/1792696254","heartbeat_front_addr":"172.21.15.144:6805/1792696254","state":["exists","up"]},{"osd":5,"uuid":"3f1ee7e1-e3ee-4351-8492-3573e3cb0fea","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":42,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6808","nonce":2187592584},{"type":"v1","addr":"172.21.15.144:6809","nonce":2187592584}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6810","nonce":2187592584},{"type":"v1","addr":"172.21.15.144:6811","nonce":2187592584}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6814","nonce":2187592584},{"type":"v1","addr":"172.21.15.144:6815","nonce":2187592584}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6812","nonce":2187592584},{"type":"v1","addr":"172.21.15.144:6813","nonce":2187592584}]},"public_addr":"172.21.15.144:6809/2187592584","cluster_addr":"172.21.15.144:6811/2187592584","heartbeat_back_addr":"172.21.15.144:6815/2187592584","heartbeat_front_addr":"172.21.15.144:6813/2187592584","state":["exists","up"]},{"osd":6,"uuid":"f19f02ff-b250-46a2-8f33-183bde556e09","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":50,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6816","nonce":3907677683},{"type":"v1","addr":"172.21.15.144:6817","nonce":3907677683}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6818","nonce":3907677683},{"type":"v1","addr":"172.21.15.144:6819","nonce":3907677683}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6822","nonce":3907677683},{"type":"v1","addr":"172.21.15.144:6823","nonce":3907677683}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6820","nonce":3907677683},{"type":"v1","addr":"172.21.15.144:6821","nonce":3907677683}]},"public_addr":"172.21.15.144:6817/3907677683","cluster_addr":"172.21.15.144:6819/3907677683","heartbeat_back_addr":"172.21.15.144:6823/3907677683","heartbeat_front_addr":"172.21.15.144:6821/3907677683","state":["exists","up"]},{"osd":7,"uuid":"812947ad-33a8-440b-882e-aacc18adc2ef","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":58,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6824","nonce":1629206793},{"type":"v1","addr":"172.21.15.144:6825","nonce":1629206793}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6826","nonce":1629206793},{"type":"v1","addr":"172.21.15.144:6827","nonce":1629206793}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6830","nonce":1629206793},{"type":"v1","addr":"172.21.15.144:6831","nonce":1629206793}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6828","nonce":1629206793},{"type":"v1","addr":"172.21.15.144:6829","nonce":1629206793}]},"public_addr":"172.21.15.144:6825/1629206793","cluster_addr":"172.21.15.144:6827/1629206793","heartbeat_back_addr":"172.21.15.144:6831/1629206793","heartbeat_front_addr":"172.21.15.144:6829/1629206793","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-16T08:08:09.177820+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-16T08:08:34.507057+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-16T08:09:02.169534+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-16T08:09:28.598100+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-16T08:09:50.082338+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-16T08:10:14.455953+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-16T08:10:38.954291+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-16T08:11:03.280241+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.28:0/3973422976":"2024-09-17T08:07:03.540703+0000","172.21.15.28:0/2703579451":"2024-09-17T08:07:03.540703+0000","172.21.15.28:6800/3593196739":"2024-09-17T08:07:03.540703+0000","172.21.15.28:0/1234187527":"2024-09-17T08:07:03.540703+0000","172.21.15.28:0/923827164":"2024-09-17T08:06:14.463773+0000","172.21.15.28:6801/24563670":"2024-09-17T08:06:14.463773+0000","172.21.15.28:6800/24563670":"2024-09-17T08:06:14.463773+0000","172.21.15.28:0/281327060":"2024-09-17T08:06:14.463773+0000","172.21.15.28:0/3672481329":"2024-09-17T08:05:45.461999+0000","172.21.15.28:0/2118149107":"2024-09-17T08:06:14.463773+0000","172.21.15.28:6801/282612086":"2024-09-17T08:05:45.461999+0000","172.21.15.28:0/3325094875":"2024-09-17T08:05:45.461999+0000","172.21.15.28:6801/3593196739":"2024-09-17T08:07:03.540703+0000","172.21.15.28:6800/282612086":"2024-09-17T08:05:45.461999+0000","172.21.15.28:0/3076918752":"2024-09-17T08:05:45.461999+0000"},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-16T08:11:22.389 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:11:22 smithi028 ceph-mon[23744]: from='client.? 172.21.15.28:0/1806045571' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-16T08:11:22.415 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2024-09-16T08:11:22.415 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 5622b2fe-7402-11ef-bceb-c7b262605968 -- ceph osd dump --format=json 2024-09-16T08:11:22.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:11:22 smithi144 ceph-mon[25668]: from='client.? 172.21.15.28:0/1806045571' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-16T08:11:22.636 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/5622b2fe-7402-11ef-bceb-c7b262605968/mon.smithi028/config 2024-09-16T08:11:23.442 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:11:23 smithi144 ceph-mon[25668]: pgmap v176: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 27 KiB/s, 0 objects/s recovering 2024-09-16T08:11:23.446 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:11:23 smithi028 ceph-mon[23744]: pgmap v176: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 27 KiB/s, 0 objects/s recovering 2024-09-16T08:11:24.342 INFO:teuthology.orchestra.run.smithi028.stdout: 2024-09-16T08:11:24.343 INFO:teuthology.orchestra.run.smithi028.stdout:{"epoch":59,"fsid":"5622b2fe-7402-11ef-bceb-c7b262605968","created":"2024-09-16T08:05:20.224194+0000","modified":"2024-09-16T08:11:09.075971+0000","last_up_change":"2024-09-16T08:11:05.067238+0000","last_in_change":"2024-09-16T08:10:47.598994+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-16T08:09:06.094185+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}}],"osds":[{"osd":0,"uuid":"66f613ac-6dba-4ca1-ab30-8b95e06260c0","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6802","nonce":230701446},{"type":"v1","addr":"172.21.15.28:6803","nonce":230701446}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6804","nonce":230701446},{"type":"v1","addr":"172.21.15.28:6805","nonce":230701446}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6808","nonce":230701446},{"type":"v1","addr":"172.21.15.28:6809","nonce":230701446}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6806","nonce":230701446},{"type":"v1","addr":"172.21.15.28:6807","nonce":230701446}]},"public_addr":"172.21.15.28:6803/230701446","cluster_addr":"172.21.15.28:6805/230701446","heartbeat_back_addr":"172.21.15.28:6809/230701446","heartbeat_front_addr":"172.21.15.28:6807/230701446","state":["exists","up"]},{"osd":1,"uuid":"524eec0a-00eb-403b-a964-ae56bd9eb6f5","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":34,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6810","nonce":3124900380},{"type":"v1","addr":"172.21.15.28:6811","nonce":3124900380}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6812","nonce":3124900380},{"type":"v1","addr":"172.21.15.28:6813","nonce":3124900380}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6816","nonce":3124900380},{"type":"v1","addr":"172.21.15.28:6817","nonce":3124900380}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6814","nonce":3124900380},{"type":"v1","addr":"172.21.15.28:6815","nonce":3124900380}]},"public_addr":"172.21.15.28:6811/3124900380","cluster_addr":"172.21.15.28:6813/3124900380","heartbeat_back_addr":"172.21.15.28:6817/3124900380","heartbeat_front_addr":"172.21.15.28:6815/3124900380","state":["exists","up"]},{"osd":2,"uuid":"cabe96b6-9d98-487a-8a56-6ef052c609f1","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":19,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6818","nonce":3138093499},{"type":"v1","addr":"172.21.15.28:6819","nonce":3138093499}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6820","nonce":3138093499},{"type":"v1","addr":"172.21.15.28:6821","nonce":3138093499}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6824","nonce":3138093499},{"type":"v1","addr":"172.21.15.28:6825","nonce":3138093499}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6822","nonce":3138093499},{"type":"v1","addr":"172.21.15.28:6823","nonce":3138093499}]},"public_addr":"172.21.15.28:6819/3138093499","cluster_addr":"172.21.15.28:6821/3138093499","heartbeat_back_addr":"172.21.15.28:6825/3138093499","heartbeat_front_addr":"172.21.15.28:6823/3138093499","state":["exists","up"]},{"osd":3,"uuid":"2400d76b-78e4-43dd-becc-d1f80427f1df","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6826","nonce":3665954147},{"type":"v1","addr":"172.21.15.28:6827","nonce":3665954147}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6828","nonce":3665954147},{"type":"v1","addr":"172.21.15.28:6829","nonce":3665954147}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6832","nonce":3665954147},{"type":"v1","addr":"172.21.15.28:6833","nonce":3665954147}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6830","nonce":3665954147},{"type":"v1","addr":"172.21.15.28:6831","nonce":3665954147}]},"public_addr":"172.21.15.28:6827/3665954147","cluster_addr":"172.21.15.28:6829/3665954147","heartbeat_back_addr":"172.21.15.28:6833/3665954147","heartbeat_front_addr":"172.21.15.28:6831/3665954147","state":["exists","up"]},{"osd":4,"uuid":"a929d7e2-fac0-47fe-9919-a4329c008de2","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6800","nonce":1792696254},{"type":"v1","addr":"172.21.15.144:6801","nonce":1792696254}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6802","nonce":1792696254},{"type":"v1","addr":"172.21.15.144:6803","nonce":1792696254}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6806","nonce":1792696254},{"type":"v1","addr":"172.21.15.144:6807","nonce":1792696254}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6804","nonce":1792696254},{"type":"v1","addr":"172.21.15.144:6805","nonce":1792696254}]},"public_addr":"172.21.15.144:6801/1792696254","cluster_addr":"172.21.15.144:6803/1792696254","heartbeat_back_addr":"172.21.15.144:6807/1792696254","heartbeat_front_addr":"172.21.15.144:6805/1792696254","state":["exists","up"]},{"osd":5,"uuid":"3f1ee7e1-e3ee-4351-8492-3573e3cb0fea","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":42,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6808","nonce":2187592584},{"type":"v1","addr":"172.21.15.144:6809","nonce":2187592584}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6810","nonce":2187592584},{"type":"v1","addr":"172.21.15.144:6811","nonce":2187592584}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6814","nonce":2187592584},{"type":"v1","addr":"172.21.15.144:6815","nonce":2187592584}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6812","nonce":2187592584},{"type":"v1","addr":"172.21.15.144:6813","nonce":2187592584}]},"public_addr":"172.21.15.144:6809/2187592584","cluster_addr":"172.21.15.144:6811/2187592584","heartbeat_back_addr":"172.21.15.144:6815/2187592584","heartbeat_front_addr":"172.21.15.144:6813/2187592584","state":["exists","up"]},{"osd":6,"uuid":"f19f02ff-b250-46a2-8f33-183bde556e09","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":50,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6816","nonce":3907677683},{"type":"v1","addr":"172.21.15.144:6817","nonce":3907677683}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6818","nonce":3907677683},{"type":"v1","addr":"172.21.15.144:6819","nonce":3907677683}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6822","nonce":3907677683},{"type":"v1","addr":"172.21.15.144:6823","nonce":3907677683}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6820","nonce":3907677683},{"type":"v1","addr":"172.21.15.144:6821","nonce":3907677683}]},"public_addr":"172.21.15.144:6817/3907677683","cluster_addr":"172.21.15.144:6819/3907677683","heartbeat_back_addr":"172.21.15.144:6823/3907677683","heartbeat_front_addr":"172.21.15.144:6821/3907677683","state":["exists","up"]},{"osd":7,"uuid":"812947ad-33a8-440b-882e-aacc18adc2ef","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":58,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6824","nonce":1629206793},{"type":"v1","addr":"172.21.15.144:6825","nonce":1629206793}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6826","nonce":1629206793},{"type":"v1","addr":"172.21.15.144:6827","nonce":1629206793}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6830","nonce":1629206793},{"type":"v1","addr":"172.21.15.144:6831","nonce":1629206793}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6828","nonce":1629206793},{"type":"v1","addr":"172.21.15.144:6829","nonce":1629206793}]},"public_addr":"172.21.15.144:6825/1629206793","cluster_addr":"172.21.15.144:6827/1629206793","heartbeat_back_addr":"172.21.15.144:6831/1629206793","heartbeat_front_addr":"172.21.15.144:6829/1629206793","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-16T08:08:09.177820+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-16T08:08:34.507057+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-16T08:09:02.169534+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-16T08:09:28.598100+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-16T08:09:50.082338+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-16T08:10:14.455953+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-16T08:10:38.954291+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-16T08:11:03.280241+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.28:0/3973422976":"2024-09-17T08:07:03.540703+0000","172.21.15.28:0/2703579451":"2024-09-17T08:07:03.540703+0000","172.21.15.28:6800/3593196739":"2024-09-17T08:07:03.540703+0000","172.21.15.28:0/1234187527":"2024-09-17T08:07:03.540703+0000","172.21.15.28:0/923827164":"2024-09-17T08:06:14.463773+0000","172.21.15.28:6801/24563670":"2024-09-17T08:06:14.463773+0000","172.21.15.28:6800/24563670":"2024-09-17T08:06:14.463773+0000","172.21.15.28:0/281327060":"2024-09-17T08:06:14.463773+0000","172.21.15.28:0/3672481329":"2024-09-17T08:05:45.461999+0000","172.21.15.28:0/2118149107":"2024-09-17T08:06:14.463773+0000","172.21.15.28:6801/282612086":"2024-09-17T08:05:45.461999+0000","172.21.15.28:0/3325094875":"2024-09-17T08:05:45.461999+0000","172.21.15.28:6801/3593196739":"2024-09-17T08:07:03.540703+0000","172.21.15.28:6800/282612086":"2024-09-17T08:05:45.461999+0000","172.21.15.28:0/3076918752":"2024-09-17T08:05:45.461999+0000"},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-16T08:11:25.014 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 5622b2fe-7402-11ef-bceb-c7b262605968 -- ceph tell osd.0 flush_pg_stats 2024-09-16T08:11:25.014 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 5622b2fe-7402-11ef-bceb-c7b262605968 -- ceph tell osd.1 flush_pg_stats 2024-09-16T08:11:25.015 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 5622b2fe-7402-11ef-bceb-c7b262605968 -- ceph tell osd.2 flush_pg_stats 2024-09-16T08:11:25.015 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 5622b2fe-7402-11ef-bceb-c7b262605968 -- ceph tell osd.3 flush_pg_stats 2024-09-16T08:11:25.016 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 5622b2fe-7402-11ef-bceb-c7b262605968 -- ceph tell osd.4 flush_pg_stats 2024-09-16T08:11:25.016 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 5622b2fe-7402-11ef-bceb-c7b262605968 -- ceph tell osd.5 flush_pg_stats 2024-09-16T08:11:25.017 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 5622b2fe-7402-11ef-bceb-c7b262605968 -- ceph tell osd.6 flush_pg_stats 2024-09-16T08:11:25.017 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 5622b2fe-7402-11ef-bceb-c7b262605968 -- ceph tell osd.7 flush_pg_stats 2024-09-16T08:11:25.446 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:11:25 smithi028 ceph-mon[23744]: pgmap v177: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-16T08:11:25.446 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:11:25 smithi028 ceph-mon[23744]: from='client.? 172.21.15.28:0/399059581' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-16T08:11:25.499 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/5622b2fe-7402-11ef-bceb-c7b262605968/mon.smithi028/config 2024-09-16T08:11:25.499 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/5622b2fe-7402-11ef-bceb-c7b262605968/mon.smithi028/config 2024-09-16T08:11:25.502 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/5622b2fe-7402-11ef-bceb-c7b262605968/mon.smithi028/config 2024-09-16T08:11:25.503 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/5622b2fe-7402-11ef-bceb-c7b262605968/mon.smithi028/config 2024-09-16T08:11:25.503 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/5622b2fe-7402-11ef-bceb-c7b262605968/mon.smithi028/config 2024-09-16T08:11:25.504 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/5622b2fe-7402-11ef-bceb-c7b262605968/mon.smithi028/config 2024-09-16T08:11:25.505 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/5622b2fe-7402-11ef-bceb-c7b262605968/mon.smithi028/config 2024-09-16T08:11:25.541 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/5622b2fe-7402-11ef-bceb-c7b262605968/mon.smithi028/config 2024-09-16T08:11:25.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:11:25 smithi144 ceph-mon[25668]: pgmap v177: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-16T08:11:25.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:11:25 smithi144 ceph-mon[25668]: from='client.? 172.21.15.28:0/399059581' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-16T08:11:27.383 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:11:27 smithi144 ceph-mon[25668]: pgmap v178: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-16T08:11:27.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:11:27 smithi028 ceph-mon[23744]: pgmap v178: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-16T08:11:29.276 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:11:29 smithi028 ceph-mon[23744]: pgmap v179: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-16T08:11:29.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:11:29 smithi144 ceph-mon[25668]: pgmap v179: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-16T08:11:31.406 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:11:31 smithi028 ceph-mon[23744]: pgmap v180: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-16T08:11:31.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:11:31 smithi144 ceph-mon[25668]: pgmap v180: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-16T08:11:33.196 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:11:33 smithi028 ceph-mon[23744]: pgmap v181: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-16T08:11:33.442 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:11:33 smithi144 ceph-mon[25668]: pgmap v181: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-16T08:11:35.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:11:35 smithi028 ceph-mon[23744]: pgmap v182: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-16T08:11:35.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:11:35 smithi144 ceph-mon[25668]: pgmap v182: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-16T08:11:36.664 INFO:teuthology.orchestra.run.smithi028.stdout:111669149724 2024-09-16T08:11:36.665 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 5622b2fe-7402-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.3 2024-09-16T08:11:37.378 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:11:37 smithi028 ceph-mon[23744]: pgmap v183: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-16T08:11:37.405 INFO:teuthology.orchestra.run.smithi028.stdout:201863462924 2024-09-16T08:11:37.406 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 5622b2fe-7402-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.6 2024-09-16T08:11:37.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:11:37 smithi144 ceph-mon[25668]: pgmap v183: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-16T08:11:37.954 INFO:teuthology.orchestra.run.smithi028.stdout:133143986199 2024-09-16T08:11:37.955 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 5622b2fe-7402-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.4 2024-09-16T08:11:38.567 INFO:teuthology.orchestra.run.smithi028.stdout:236223201289 2024-09-16T08:11:38.567 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 5622b2fe-7402-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.7 2024-09-16T08:11:39.419 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:11:39 smithi028 ceph-mon[23744]: pgmap v184: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-16T08:11:39.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:11:39 smithi144 ceph-mon[25668]: pgmap v184: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-16T08:11:39.832 INFO:teuthology.orchestra.run.smithi028.stdout:81604378656 2024-09-16T08:11:39.833 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 5622b2fe-7402-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.2 2024-09-16T08:11:40.232 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/5622b2fe-7402-11ef-bceb-c7b262605968/mon.smithi028/config 2024-09-16T08:11:40.232 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/5622b2fe-7402-11ef-bceb-c7b262605968/mon.smithi028/config 2024-09-16T08:11:40.922 INFO:teuthology.orchestra.run.smithi028.stdout:38654705709 2024-09-16T08:11:40.923 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 5622b2fe-7402-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.0 2024-09-16T08:11:41.196 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:11:41 smithi028 ceph-mon[23744]: pgmap v185: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-16T08:11:41.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:11:41 smithi144 ceph-mon[25668]: pgmap v185: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-16T08:11:41.709 INFO:teuthology.orchestra.run.smithi028.stdout:60129542184 2024-09-16T08:11:41.709 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 5622b2fe-7402-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.1 2024-09-16T08:11:41.861 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/5622b2fe-7402-11ef-bceb-c7b262605968/mon.smithi028/config 2024-09-16T08:11:41.861 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/5622b2fe-7402-11ef-bceb-c7b262605968/mon.smithi028/config 2024-09-16T08:11:41.862 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/5622b2fe-7402-11ef-bceb-c7b262605968/mon.smithi028/config 2024-09-16T08:11:43.441 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:11:43 smithi144 ceph-mon[25668]: pgmap v186: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-16T08:11:43.446 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:11:43 smithi028 ceph-mon[23744]: pgmap v186: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-16T08:11:43.593 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/5622b2fe-7402-11ef-bceb-c7b262605968/mon.smithi028/config 2024-09-16T08:11:43.788 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/5622b2fe-7402-11ef-bceb-c7b262605968/mon.smithi028/config 2024-09-16T08:11:44.169 INFO:teuthology.orchestra.run.smithi028.stdout:167503724563 2024-09-16T08:11:44.170 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 5622b2fe-7402-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.5 2024-09-16T08:11:44.567 INFO:teuthology.orchestra.run.smithi028.stdout:133143986201 2024-09-16T08:11:45.153 INFO:teuthology.orchestra.run.smithi028.stdout:111669149726 2024-09-16T08:11:45.294 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:11:45 smithi028 ceph-mon[23744]: pgmap v187: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-16T08:11:45.295 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:11:45 smithi028 ceph-mon[23744]: from='client.? 172.21.15.28:0/2023175712' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-09-16T08:11:45.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:11:45 smithi144 ceph-mon[25668]: pgmap v187: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-16T08:11:45.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:11:45 smithi144 ceph-mon[25668]: from='client.? 172.21.15.28:0/2023175712' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-09-16T08:11:46.446 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:11:46 smithi028 ceph-mon[23744]: from='client.? 172.21.15.28:0/3863264916' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-09-16T08:11:46.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:11:46 smithi144 ceph-mon[25668]: from='client.? 172.21.15.28:0/3863264916' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-09-16T08:11:47.010 INFO:tasks.cephadm.ceph_manager.ceph:need seq 133143986199 got 133143986201 for osd.4 2024-09-16T08:11:47.010 DEBUG:teuthology.parallel:result is None 2024-09-16T08:11:47.274 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:11:47 smithi028 ceph-mon[23744]: pgmap v188: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-16T08:11:47.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:11:47 smithi144 ceph-mon[25668]: pgmap v188: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-16T08:11:49.196 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:11:49 smithi028 ceph-mon[23744]: pgmap v189: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-16T08:11:49.349 INFO:teuthology.orchestra.run.smithi028.stdout:236223201291 2024-09-16T08:11:49.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:11:49 smithi144 ceph-mon[25668]: pgmap v189: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-16T08:11:49.660 INFO:tasks.cephadm.ceph_manager.ceph:need seq 111669149724 got 111669149726 for osd.3 2024-09-16T08:11:49.661 DEBUG:teuthology.parallel:result is None 2024-09-16T08:11:50.291 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:11:50 smithi028 ceph-mon[23744]: from='client.? 172.21.15.28:0/975756318' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-09-16T08:11:50.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:11:50 smithi144 ceph-mon[25668]: from='client.? 172.21.15.28:0/975756318' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-09-16T08:11:50.842 INFO:teuthology.orchestra.run.smithi028.stdout:81604378659 2024-09-16T08:11:51.446 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:11:51 smithi028 ceph-mon[23744]: pgmap v190: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-16T08:11:51.446 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:11:51 smithi028 ceph-mon[23744]: from='client.? 172.21.15.28:0/4256095844' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-09-16T08:11:51.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:11:51 smithi144 ceph-mon[25668]: pgmap v190: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-16T08:11:51.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:11:51 smithi144 ceph-mon[25668]: from='client.? 172.21.15.28:0/4256095844' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-09-16T08:11:51.708 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/5622b2fe-7402-11ef-bceb-c7b262605968/mon.smithi028/config 2024-09-16T08:11:52.168 INFO:tasks.cephadm.ceph_manager.ceph:need seq 81604378656 got 81604378659 for osd.2 2024-09-16T08:11:52.168 DEBUG:teuthology.parallel:result is None 2024-09-16T08:11:52.538 INFO:tasks.cephadm.ceph_manager.ceph:need seq 236223201289 got 236223201291 for osd.7 2024-09-16T08:11:52.538 DEBUG:teuthology.parallel:result is None 2024-09-16T08:11:53.401 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:11:53 smithi028 ceph-mon[23744]: pgmap v191: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-16T08:11:53.441 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:11:53 smithi144 ceph-mon[25668]: pgmap v191: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-16T08:11:53.999 INFO:teuthology.orchestra.run.smithi028.stdout:38654705711 2024-09-16T08:11:54.417 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:11:54 smithi028 ceph-mon[23744]: from='client.? 172.21.15.28:0/2999883766' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-09-16T08:11:54.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:11:54 smithi144 ceph-mon[25668]: from='client.? 172.21.15.28:0/2999883766' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-09-16T08:11:54.947 INFO:teuthology.orchestra.run.smithi028.stdout:201863462929 2024-09-16T08:11:55.168 INFO:tasks.cephadm.ceph_manager.ceph:need seq 38654705709 got 38654705711 for osd.0 2024-09-16T08:11:55.168 DEBUG:teuthology.parallel:result is None 2024-09-16T08:11:55.446 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:11:55 smithi028 ceph-mon[23744]: pgmap v192: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-16T08:11:55.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:11:55 smithi028 ceph-mon[23744]: from='client.? 172.21.15.28:0/724626102' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-09-16T08:11:55.564 INFO:teuthology.orchestra.run.smithi028.stdout:60129542187 2024-09-16T08:11:55.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:11:55 smithi144 ceph-mon[25668]: pgmap v192: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-16T08:11:55.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:11:55 smithi144 ceph-mon[25668]: from='client.? 172.21.15.28:0/724626102' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-09-16T08:11:55.854 INFO:tasks.cephadm.ceph_manager.ceph:need seq 201863462924 got 201863462929 for osd.6 2024-09-16T08:11:55.854 DEBUG:teuthology.parallel:result is None 2024-09-16T08:11:56.446 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:11:56 smithi028 ceph-mon[23744]: from='client.? 172.21.15.28:0/1870814711' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-09-16T08:11:56.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:11:56 smithi144 ceph-mon[25668]: from='client.? 172.21.15.28:0/1870814711' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-09-16T08:11:57.161 INFO:tasks.cephadm.ceph_manager.ceph:need seq 60129542184 got 60129542187 for osd.1 2024-09-16T08:11:57.162 DEBUG:teuthology.parallel:result is None 2024-09-16T08:11:57.382 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:11:57 smithi028 ceph-mon[23744]: pgmap v193: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-16T08:11:57.482 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:11:57 smithi144 ceph-mon[25668]: pgmap v193: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-16T08:11:57.600 INFO:teuthology.orchestra.run.smithi028.stdout:167503724566 2024-09-16T08:11:58.180 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:11:58 smithi028 ceph-mon[23744]: from='client.? 172.21.15.28:0/3486540070' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-09-16T08:11:58.204 INFO:tasks.cephadm.ceph_manager.ceph:need seq 167503724563 got 167503724566 for osd.5 2024-09-16T08:11:58.204 DEBUG:teuthology.parallel:result is None 2024-09-16T08:11:58.204 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2024-09-16T08:11:58.205 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 5622b2fe-7402-11ef-bceb-c7b262605968 -- ceph pg dump --format=json 2024-09-16T08:11:58.425 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/5622b2fe-7402-11ef-bceb-c7b262605968/mon.smithi028/config 2024-09-16T08:11:58.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:11:58 smithi144 ceph-mon[25668]: from='client.? 172.21.15.28:0/3486540070' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-09-16T08:11:59.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:11:59 smithi028 ceph-mon[23744]: pgmap v194: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-16T08:11:59.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:11:59 smithi144 ceph-mon[25668]: pgmap v194: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-16T08:12:00.174 INFO:teuthology.orchestra.run.smithi028.stdout: 2024-09-16T08:12:00.175 INFO:teuthology.orchestra.run.smithi028.stderr:dumped all 2024-09-16T08:12:00.831 INFO:teuthology.orchestra.run.smithi028.stdout:{"pg_ready":true,"pg_map":{"version":195,"stamp":"2024-09-16T08:11:59.644093+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":117,"ondisk_log_size":117,"up":3,"acting":3,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":3,"num_osds":8,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":749961216,"kb_used":48812,"kb_used_data":6316,"kb_used_omap":0,"kb_used_meta":42432,"kb_avail":749912404,"statfs":{"total":767960285184,"available":767910301696,"internally_reserved":0,"allocated":6467584,"data_stored":3547795,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":43450368},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002463"},"pg_stats":[{"pgid":"1.0","version":"22'117","reported_seq":226,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-16T08:11:09.082247+0000","last_change":"2024-09-16T08:11:09.082247+0000","last_active":"2024-09-16T08:11:09.082247+0000","last_peered":"2024-09-16T08:11:09.082247+0000","last_clean":"2024-09-16T08:11:09.082247+0000","last_became_active":"2024-09-16T08:11:09.081812+0000","last_became_peered":"2024-09-16T08:11:09.081812+0000","last_unstale":"2024-09-16T08:11:09.082247+0000","last_undegraded":"2024-09-16T08:11:09.082247+0000","last_fullsized":"2024-09-16T08:11:09.082247+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-16T08:09:06.094292+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-16T08:09:06.094292+0000","last_clean_scrub_stamp":"2024-09-16T08:09:06.094292+0000","objects_scrubbed":0,"log_size":117,"ondisk_log_size":117,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-17T09:29:48.315968+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]}],"pool_stats":[{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":3727360,"data_stored":3702496,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":117,"ondisk_log_size":117,"up":3,"acting":3,"num_store_stats":7}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201293,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6052,"kb_used_data":1116,"kb_used_omap":0,"kb_used_meta":4928,"kb_avail":93739100,"statfs":{"total":95995035648,"available":95988838400,"internally_reserved":0,"allocated":1142784,"data_stored":774161,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5046272},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.60099999999999998}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.56200000000000006}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.53700000000000003}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.61099999999999999}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.64800000000000002}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.63100000000000001}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.67500000000000004}]}]},{"osd":6,"up_from":47,"seq":201863462930,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6116,"kb_used_data":1116,"kb_used_omap":0,"kb_used_meta":4992,"kb_avail":93739036,"statfs":{"total":95995035648,"available":95988772864,"internally_reserved":0,"allocated":1142784,"data_stored":774161,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5111808},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Sep 16 08:11:40 2024","interfaces":[{"interface":"back","average":{"1min":0.51800000000000002,"5min":0.51800000000000002,"15min":0.51800000000000002},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":0.72299999999999998,"5min":0.72299999999999998,"15min":0.72299999999999998},"last":0.60599999999999998},{"interface":"front","average":{"1min":0.53600000000000003,"5min":0.53600000000000003,"15min":0.53600000000000003},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":0.68200000000000005,"5min":0.68200000000000005,"15min":0.68200000000000005},"last":0.64700000000000002}]},{"osd":1,"last update":"Mon Sep 16 08:11:40 2024","interfaces":[{"interface":"back","average":{"1min":0.55400000000000005,"5min":0.55400000000000005,"15min":0.55400000000000005},"min":{"1min":0.373,"5min":0.373,"15min":0.373},"max":{"1min":0.746,"5min":0.746,"15min":0.746},"last":0.55300000000000005},{"interface":"front","average":{"1min":0.54400000000000004,"5min":0.54400000000000004,"15min":0.54400000000000004},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":0.83199999999999996,"5min":0.83199999999999996,"15min":0.83199999999999996},"last":0.56999999999999995}]},{"osd":2,"last update":"Mon Sep 16 08:11:40 2024","interfaces":[{"interface":"back","average":{"1min":0.54500000000000004,"5min":0.54500000000000004,"15min":0.54500000000000004},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":0.78100000000000003,"5min":0.78100000000000003,"15min":0.78100000000000003},"last":0.51300000000000001},{"interface":"front","average":{"1min":0.57199999999999995,"5min":0.57199999999999995,"15min":0.57199999999999995},"min":{"1min":0.41099999999999998,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":0.755,"5min":0.755,"15min":0.755},"last":0.58299999999999996}]},{"osd":3,"last update":"Mon Sep 16 08:11:40 2024","interfaces":[{"interface":"back","average":{"1min":0.55300000000000005,"5min":0.55300000000000005,"15min":0.55300000000000005},"min":{"1min":0.40600000000000003,"5min":0.40600000000000003,"15min":0.40600000000000003},"max":{"1min":0.68500000000000005,"5min":0.68500000000000005,"15min":0.68500000000000005},"last":0.63},{"interface":"front","average":{"1min":0.56699999999999995,"5min":0.56699999999999995,"15min":0.56699999999999995},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":0.80800000000000005,"5min":0.80800000000000005,"15min":0.80800000000000005},"last":0.623}]},{"osd":4,"last update":"Mon Sep 16 08:11:40 2024","interfaces":[{"interface":"back","average":{"1min":0.47499999999999998,"5min":0.47499999999999998,"15min":0.47499999999999998},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":0.74099999999999999,"5min":0.74099999999999999,"15min":0.74099999999999999},"last":0.61499999999999999},{"interface":"front","average":{"1min":0.45800000000000002,"5min":0.45800000000000002,"15min":0.45800000000000002},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":0.75700000000000001,"5min":0.75700000000000001,"15min":0.75700000000000001},"last":0.59799999999999998}]},{"osd":5,"last update":"Mon Sep 16 08:11:40 2024","interfaces":[{"interface":"back","average":{"1min":0.48599999999999999,"5min":0.48599999999999999,"15min":0.48599999999999999},"min":{"1min":0.311,"5min":0.311,"15min":0.311},"max":{"1min":0.81599999999999995,"5min":0.81599999999999995,"15min":0.81599999999999995},"last":0.53300000000000003},{"interface":"front","average":{"1min":0.505,"5min":0.505,"15min":0.505},"min":{"1min":0.317,"5min":0.317,"15min":0.317},"max":{"1min":0.751,"5min":0.751,"15min":0.751},"last":0.70899999999999996}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.67300000000000004}]}]},{"osd":1,"up_from":14,"seq":60129542188,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":6360,"kb_used_data":592,"kb_used_omap":0,"kb_used_meta":5760,"kb_avail":93738792,"statfs":{"total":95995035648,"available":95988523008,"internally_reserved":0,"allocated":606208,"data_stored":244968,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5898240},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Sep 16 08:11:46 2024","interfaces":[{"interface":"back","average":{"1min":0.39600000000000002,"5min":0.36099999999999999,"15min":0.33700000000000002},"min":{"1min":0.28599999999999998,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":0.57699999999999996,"5min":0.63500000000000001,"15min":0.63500000000000001},"last":0.35099999999999998},{"interface":"front","average":{"1min":0.43099999999999999,"5min":0.379,"15min":0.35499999999999998},"min":{"1min":0.27700000000000002,"5min":0.214,"15min":0.214},"max":{"1min":0.64200000000000002,"5min":0.64200000000000002,"15min":0.64200000000000002},"last":0.50700000000000001}]},{"osd":2,"last update":"Mon Sep 16 08:11:11 2024","interfaces":[{"interface":"back","average":{"1min":0.46400000000000002,"5min":0.41099999999999998,"15min":0.40200000000000002},"min":{"1min":0.318,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":0.65700000000000003,"5min":0.65700000000000003,"15min":0.65700000000000003},"last":0.44600000000000001},{"interface":"front","average":{"1min":0.42799999999999999,"5min":0.41199999999999998,"15min":0.40899999999999997},"min":{"1min":0.33700000000000002,"5min":0.31,"15min":0.31},"max":{"1min":0.59299999999999997,"5min":0.61699999999999999,"15min":0.61699999999999999},"last":0.41299999999999998}]},{"osd":3,"last update":"Mon Sep 16 08:11:36 2024","interfaces":[{"interface":"back","average":{"1min":0.433,"5min":0.45900000000000002,"15min":0.46300000000000002},"min":{"1min":0.314,"5min":0.314,"15min":0.314},"max":{"1min":0.60299999999999998,"5min":0.60299999999999998,"15min":0.60299999999999998},"last":0.46400000000000002},{"interface":"front","average":{"1min":0.48199999999999998,"5min":0.49199999999999999,"15min":0.49399999999999999},"min":{"1min":0.36699999999999999,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":0.68999999999999995,"5min":0.69999999999999996,"15min":0.69999999999999996},"last":0.49199999999999999}]},{"osd":4,"last update":"Mon Sep 16 08:10:53 2024","interfaces":[{"interface":"back","average":{"1min":0.63700000000000001,"5min":0.63700000000000001,"15min":0.63700000000000001},"min":{"1min":0.47099999999999997,"5min":0.47099999999999997,"15min":0.47099999999999997},"max":{"1min":0.81200000000000006,"5min":0.81200000000000006,"15min":0.81200000000000006},"last":0.66000000000000003},{"interface":"front","average":{"1min":0.63300000000000001,"5min":0.63300000000000001,"15min":0.63300000000000001},"min":{"1min":0.48199999999999998,"5min":0.48199999999999998,"15min":0.48199999999999998},"max":{"1min":0.84399999999999997,"5min":0.84399999999999997,"15min":0.84399999999999997},"last":0.66500000000000004}]},{"osd":5,"last update":"Mon Sep 16 08:11:21 2024","interfaces":[{"interface":"back","average":{"1min":0.61799999999999999,"5min":0.61799999999999999,"15min":0.61799999999999999},"min":{"1min":0.40300000000000002,"5min":0.40300000000000002,"15min":0.40300000000000002},"max":{"1min":0.85699999999999998,"5min":0.85699999999999998,"15min":0.85699999999999998},"last":0.56899999999999995},{"interface":"front","average":{"1min":0.64200000000000002,"5min":0.64200000000000002,"15min":0.64200000000000002},"min":{"1min":0.433,"5min":0.433,"15min":0.433},"max":{"1min":0.80100000000000005,"5min":0.80100000000000005,"15min":0.80100000000000005},"last":0.60899999999999999}]},{"osd":6,"last update":"Mon Sep 16 08:11:44 2024","interfaces":[{"interface":"back","average":{"1min":0.66400000000000003,"5min":0.66400000000000003,"15min":0.66400000000000003},"min":{"1min":0.40100000000000002,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":0.98999999999999999,"5min":0.98999999999999999,"15min":0.98999999999999999},"last":0.55600000000000005},{"interface":"front","average":{"1min":0.65000000000000002,"5min":0.65000000000000002,"15min":0.65000000000000002},"min":{"1min":0.46600000000000003,"5min":0.46600000000000003,"15min":0.46600000000000003},"max":{"1min":0.88600000000000001,"5min":0.88600000000000001,"15min":0.88600000000000001},"last":0.59799999999999998}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.57999999999999996}]}]},{"osd":0,"up_from":9,"seq":38654705713,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6884,"kb_used_data":1116,"kb_used_omap":0,"kb_used_meta":5760,"kb_avail":93738268,"statfs":{"total":95995035648,"available":95987986432,"internally_reserved":0,"allocated":1142784,"data_stored":774161,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5898240},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Sep 16 08:11:46 2024","interfaces":[{"interface":"back","average":{"1min":0.35299999999999998,"5min":0.33400000000000002,"15min":0.316},"min":{"1min":0.23100000000000001,"5min":0.19500000000000001,"15min":0.19500000000000001},"max":{"1min":0.53900000000000003,"5min":0.55100000000000005,"15min":0.55100000000000005},"last":0.33500000000000002},{"interface":"front","average":{"1min":0.41099999999999998,"5min":0.36699999999999999,"15min":0.34100000000000003},"min":{"1min":0.26500000000000001,"5min":0.187,"15min":0.187},"max":{"1min":0.53800000000000003,"5min":0.56899999999999995,"15min":0.56899999999999995},"last":0.42199999999999999}]},{"osd":2,"last update":"Mon Sep 16 08:11:08 2024","interfaces":[{"interface":"back","average":{"1min":0.42599999999999999,"5min":0.42499999999999999,"15min":0.42499999999999999},"min":{"1min":0.24099999999999999,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":0.63300000000000001,"5min":0.63300000000000001,"15min":0.63300000000000001},"last":0.47699999999999998},{"interface":"front","average":{"1min":0.45400000000000001,"5min":0.442,"15min":0.44},"min":{"1min":0.28399999999999997,"5min":0.224,"15min":0.224},"max":{"1min":0.67400000000000004,"5min":0.68999999999999995,"15min":0.68999999999999995},"last":0.34999999999999998}]},{"osd":3,"last update":"Mon Sep 16 08:11:34 2024","interfaces":[{"interface":"back","average":{"1min":0.44500000000000001,"5min":0.48499999999999999,"15min":0.49099999999999999},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":0.59099999999999997,"5min":0.72799999999999998,"15min":0.72799999999999998},"last":0.45600000000000002},{"interface":"front","average":{"1min":0.45700000000000002,"5min":0.47999999999999998,"15min":0.48399999999999999},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":0.63900000000000001,"5min":0.64500000000000002,"15min":0.64500000000000002},"last":0.44700000000000001}]},{"osd":4,"last update":"Mon Sep 16 08:10:56 2024","interfaces":[{"interface":"back","average":{"1min":0.61199999999999999,"5min":0.61199999999999999,"15min":0.61199999999999999},"min":{"1min":0.40600000000000003,"5min":0.40600000000000003,"15min":0.40600000000000003},"max":{"1min":0.79900000000000004,"5min":0.79900000000000004,"15min":0.79900000000000004},"last":0.70099999999999996},{"interface":"front","average":{"1min":0.626,"5min":0.626,"15min":0.626},"min":{"1min":0.44400000000000001,"5min":0.44400000000000001,"15min":0.44400000000000001},"max":{"1min":0.90300000000000002,"5min":0.90300000000000002,"15min":0.90300000000000002},"last":0.71199999999999997}]},{"osd":5,"last update":"Mon Sep 16 08:11:16 2024","interfaces":[{"interface":"back","average":{"1min":0.63200000000000001,"5min":0.63200000000000001,"15min":0.63200000000000001},"min":{"1min":0.46600000000000003,"5min":0.46600000000000003,"15min":0.46600000000000003},"max":{"1min":0.81100000000000005,"5min":0.81100000000000005,"15min":0.81100000000000005},"last":0.67200000000000004},{"interface":"front","average":{"1min":0.63900000000000001,"5min":0.63900000000000001,"15min":0.63900000000000001},"min":{"1min":0.52100000000000002,"5min":0.52100000000000002,"15min":0.52100000000000002},"max":{"1min":0.82299999999999995,"5min":0.82299999999999995,"15min":0.82299999999999995},"last":0.621}]},{"osd":6,"last update":"Mon Sep 16 08:11:40 2024","interfaces":[{"interface":"back","average":{"1min":0.61399999999999999,"5min":0.61399999999999999,"15min":0.61399999999999999},"min":{"1min":0.434,"5min":0.434,"15min":0.434},"max":{"1min":0.79500000000000004,"5min":0.79500000000000004,"15min":0.79500000000000004},"last":0.72599999999999998},{"interface":"front","average":{"1min":0.64400000000000002,"5min":0.64400000000000002,"15min":0.64400000000000002},"min":{"1min":0.45600000000000002,"5min":0.45600000000000002,"15min":0.45600000000000002},"max":{"1min":0.90500000000000003,"5min":0.90500000000000003,"15min":0.90500000000000003},"last":0.64400000000000002}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.63100000000000001}]}]},{"osd":2,"up_from":19,"seq":81604378661,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":6300,"kb_used_data":596,"kb_used_omap":0,"kb_used_meta":5696,"kb_avail":93738852,"statfs":{"total":95995035648,"available":95988584448,"internally_reserved":0,"allocated":610304,"data_stored":245233,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5832704},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Sep 16 08:11:08 2024","interfaces":[{"interface":"back","average":{"1min":0.43099999999999999,"5min":0.378,"15min":0.36899999999999999},"min":{"1min":0.314,"5min":0.221,"15min":0.221},"max":{"1min":0.67900000000000005,"5min":0.67900000000000005,"15min":0.67900000000000005},"last":0.496},{"interface":"front","average":{"1min":0.44500000000000001,"5min":0.378,"15min":0.36699999999999999},"min":{"1min":0.29099999999999998,"5min":0.245,"15min":0.245},"max":{"1min":0.59199999999999997,"5min":0.59199999999999997,"15min":0.59199999999999997},"last":0.52900000000000003}]},{"osd":1,"last update":"Mon Sep 16 08:11:08 2024","interfaces":[{"interface":"back","average":{"1min":0.45700000000000002,"5min":0.38900000000000001,"15min":0.377},"min":{"1min":0.33200000000000002,"5min":0.252,"15min":0.252},"max":{"1min":0.56799999999999995,"5min":0.72999999999999998,"15min":0.72999999999999998},"last":0.46600000000000003},{"interface":"front","average":{"1min":0.45700000000000002,"5min":0.377,"15min":0.36399999999999999},"min":{"1min":0.30299999999999999,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":0.70399999999999996,"5min":0.70399999999999996,"15min":0.70399999999999996},"last":0.51400000000000001}]},{"osd":3,"last update":"Mon Sep 16 08:11:31 2024","interfaces":[{"interface":"back","average":{"1min":0.39600000000000002,"5min":0.45000000000000001,"15min":0.45900000000000002},"min":{"1min":0.29099999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":0.60499999999999998,"5min":0.64900000000000002,"15min":0.64900000000000002},"last":0.48099999999999998},{"interface":"front","average":{"1min":0.47499999999999998,"5min":0.504,"15min":0.50900000000000001},"min":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":0.64100000000000001,"5min":0.71199999999999997,"15min":0.71199999999999997},"last":0.64700000000000002}]},{"osd":4,"last update":"Mon Sep 16 08:10:54 2024","interfaces":[{"interface":"back","average":{"1min":0.64800000000000002,"5min":0.64800000000000002,"15min":0.64800000000000002},"min":{"1min":0.46400000000000002,"5min":0.46400000000000002,"15min":0.46400000000000002},"max":{"1min":0.871,"5min":0.871,"15min":0.871},"last":0.88400000000000001},{"interface":"front","average":{"1min":0.63800000000000001,"5min":0.63800000000000001,"15min":0.63800000000000001},"min":{"1min":0.46800000000000003,"5min":0.46800000000000003,"15min":0.46800000000000003},"max":{"1min":0.84199999999999997,"5min":0.84199999999999997,"15min":0.84199999999999997},"last":0.90200000000000002}]},{"osd":5,"last update":"Mon Sep 16 08:11:17 2024","interfaces":[{"interface":"back","average":{"1min":0.59399999999999997,"5min":0.59399999999999997,"15min":0.59399999999999997},"min":{"1min":0.39900000000000002,"5min":0.39900000000000002,"15min":0.39900000000000002},"max":{"1min":0.876,"5min":0.876,"15min":0.876},"last":0.98099999999999998},{"interface":"front","average":{"1min":0.60699999999999998,"5min":0.60699999999999998,"15min":0.60699999999999998},"min":{"1min":0.44400000000000001,"5min":0.44400000000000001,"15min":0.44400000000000001},"max":{"1min":0.81699999999999995,"5min":0.81699999999999995,"15min":0.81699999999999995},"last":0.89100000000000001}]},{"osd":6,"last update":"Mon Sep 16 08:11:42 2024","interfaces":[{"interface":"back","average":{"1min":0.621,"5min":0.621,"15min":0.621},"min":{"1min":0.46700000000000003,"5min":0.46700000000000003,"15min":0.46700000000000003},"max":{"1min":0.83299999999999996,"5min":0.83299999999999996,"15min":0.83299999999999996},"last":0.82099999999999995},{"interface":"front","average":{"1min":0.61899999999999999,"5min":0.61899999999999999,"15min":0.61899999999999999},"min":{"1min":0.433,"5min":0.433,"15min":0.433},"max":{"1min":0.84099999999999997,"5min":0.84099999999999997,"15min":0.84099999999999997},"last":0.93799999999999994}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.86299999999999999}]}]},{"osd":3,"up_from":26,"seq":111669149729,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5720,"kb_used_data":592,"kb_used_omap":0,"kb_used_meta":5120,"kb_avail":93739432,"statfs":{"total":95995035648,"available":95989178368,"internally_reserved":0,"allocated":606208,"data_stored":244968,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5242880},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Sep 16 08:11:32 2024","interfaces":[{"interface":"back","average":{"1min":0.41699999999999998,"5min":0.40000000000000002,"15min":0.39700000000000002},"min":{"1min":0.253,"5min":0.253,"15min":0.253},"max":{"1min":1.0980000000000001,"5min":1.0980000000000001,"15min":1.0980000000000001},"last":0.35699999999999998},{"interface":"front","average":{"1min":0.42199999999999999,"5min":0.39200000000000002,"15min":0.38700000000000001},"min":{"1min":0.20399999999999999,"5min":0.20399999999999999,"15min":0.20399999999999999},"max":{"1min":0.61699999999999999,"5min":0.61699999999999999,"15min":0.61699999999999999},"last":0.45400000000000001}]},{"osd":1,"last update":"Mon Sep 16 08:11:32 2024","interfaces":[{"interface":"back","average":{"1min":0.41599999999999998,"5min":0.41199999999999998,"15min":0.41099999999999998},"min":{"1min":0.25800000000000001,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":0.58399999999999996,"5min":0.58399999999999996,"15min":0.58399999999999996},"last":0.40600000000000003},{"interface":"front","average":{"1min":0.40899999999999997,"5min":0.39700000000000002,"15min":0.39500000000000002},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":0.70999999999999996,"5min":0.70999999999999996,"15min":0.70999999999999996},"last":0.374}]},{"osd":2,"last update":"Mon Sep 16 08:11:32 2024","interfaces":[{"interface":"back","average":{"1min":0.47799999999999998,"5min":0.48599999999999999,"15min":0.48799999999999999},"min":{"1min":0.34999999999999998,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":0.69399999999999995,"5min":0.70299999999999996,"15min":0.70299999999999996},"last":0.42299999999999999},{"interface":"front","average":{"1min":0.42999999999999999,"5min":0.46600000000000003,"15min":0.47199999999999998},"min":{"1min":0.217,"5min":0.217,"15min":0.217},"max":{"1min":0.66000000000000003,"5min":0.66000000000000003,"15min":0.66000000000000003},"last":0.433}]},{"osd":4,"last update":"Mon Sep 16 08:11:52 2024","interfaces":[{"interface":"back","average":{"1min":0.59399999999999997,"5min":0.56699999999999995,"15min":0.56299999999999994},"min":{"1min":0.42099999999999999,"5min":0.42099999999999999,"15min":0.42099999999999999},"max":{"1min":0.98999999999999999,"5min":0.98999999999999999,"15min":0.98999999999999999},"last":0.63900000000000001},{"interface":"front","average":{"1min":0.60599999999999998,"5min":0.59199999999999997,"15min":0.58999999999999997},"min":{"1min":0.46999999999999997,"5min":0.443,"15min":0.443},"max":{"1min":0.85099999999999998,"5min":0.86099999999999999,"15min":0.86099999999999999},"last":0.629}]},{"osd":5,"last update":"Mon Sep 16 08:11:19 2024","interfaces":[{"interface":"back","average":{"1min":0.58399999999999996,"5min":0.58399999999999996,"15min":0.58399999999999996},"min":{"1min":0.40999999999999998,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":0.85299999999999998,"5min":0.85299999999999998,"15min":0.85299999999999998},"last":0.66300000000000003},{"interface":"front","average":{"1min":0.59399999999999997,"5min":0.59399999999999997,"15min":0.59399999999999997},"min":{"1min":0.438,"5min":0.438,"15min":0.438},"max":{"1min":0.79900000000000004,"5min":0.79900000000000004,"15min":0.79900000000000004},"last":0.54400000000000004}]},{"osd":6,"last update":"Mon Sep 16 08:11:45 2024","interfaces":[{"interface":"back","average":{"1min":0.63300000000000001,"5min":0.63300000000000001,"15min":0.63300000000000001},"min":{"1min":0.42499999999999999,"5min":0.42499999999999999,"15min":0.42499999999999999},"max":{"1min":0.90100000000000002,"5min":0.90100000000000002,"15min":0.90100000000000002},"last":0.56499999999999995},{"interface":"front","average":{"1min":0.63,"5min":0.63,"15min":0.63},"min":{"1min":0.46400000000000002,"5min":0.46400000000000002,"15min":0.46400000000000002},"max":{"1min":0.88500000000000001,"5min":0.88500000000000001,"15min":0.88500000000000001},"last":0.65200000000000002}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.58899999999999997}]}]},{"osd":4,"up_from":31,"seq":133143986204,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5724,"kb_used_data":596,"kb_used_omap":0,"kb_used_meta":5120,"kb_avail":93739428,"statfs":{"total":95995035648,"available":95989174272,"internally_reserved":0,"allocated":610304,"data_stored":245233,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5242880},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Sep 16 08:11:55 2024","interfaces":[{"interface":"back","average":{"1min":0.57299999999999995,"5min":0.56899999999999995,"15min":0.56799999999999995},"min":{"1min":0.33900000000000002,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":0.82599999999999996,"5min":0.82599999999999996,"15min":0.82599999999999996},"last":0.55600000000000005},{"interface":"front","average":{"1min":0.55300000000000005,"5min":0.57399999999999995,"15min":0.57799999999999996},"min":{"1min":0.39000000000000001,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":0.70999999999999996,"5min":0.84799999999999998,"15min":0.84799999999999998},"last":0.53500000000000003}]},{"osd":1,"last update":"Mon Sep 16 08:11:55 2024","interfaces":[{"interface":"back","average":{"1min":0.53500000000000003,"5min":0.56799999999999995,"15min":0.57399999999999995},"min":{"1min":0.34899999999999998,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":0.71299999999999997,"5min":0.80800000000000005,"15min":0.80800000000000005},"last":0.499},{"interface":"front","average":{"1min":0.57099999999999995,"5min":0.56299999999999994,"15min":0.56200000000000006},"min":{"1min":0.38200000000000001,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":0.71699999999999997,"5min":0.874,"15min":0.874},"last":0.59299999999999997}]},{"osd":2,"last update":"Mon Sep 16 08:11:55 2024","interfaces":[{"interface":"back","average":{"1min":0.59899999999999998,"5min":0.57699999999999996,"15min":0.57299999999999995},"min":{"1min":0.46899999999999997,"5min":0.318,"15min":0.318},"max":{"1min":0.746,"5min":0.81599999999999995,"15min":0.81599999999999995},"last":0.51900000000000002},{"interface":"front","average":{"1min":0.56999999999999995,"5min":0.57599999999999996,"15min":0.57699999999999996},"min":{"1min":0.44,"5min":0.379,"15min":0.379},"max":{"1min":0.754,"5min":0.94999999999999996,"15min":0.94999999999999996},"last":0.57799999999999996}]},{"osd":3,"last update":"Mon Sep 16 08:11:55 2024","interfaces":[{"interface":"back","average":{"1min":0.56100000000000005,"5min":0.58499999999999996,"15min":0.58899999999999997},"min":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":0.79100000000000004,"5min":0.91900000000000004,"15min":0.91900000000000004},"last":0.38500000000000001},{"interface":"front","average":{"1min":0.57299999999999995,"5min":0.56999999999999995,"15min":0.56999999999999995},"min":{"1min":0.39300000000000002,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":0.78100000000000003,"5min":0.84599999999999997,"15min":0.84599999999999997},"last":0.41499999999999998}]},{"osd":5,"last update":"Mon Sep 16 08:11:17 2024","interfaces":[{"interface":"back","average":{"1min":0.40300000000000002,"5min":0.40300000000000002,"15min":0.40300000000000002},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":0.69699999999999995,"5min":0.69699999999999995,"15min":0.69699999999999995},"last":0.46800000000000003},{"interface":"front","average":{"1min":0.41099999999999998,"5min":0.41099999999999998,"15min":0.41099999999999998},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":0.71699999999999997,"5min":0.71699999999999997,"15min":0.71699999999999997},"last":0.48299999999999998}]},{"osd":6,"last update":"Mon Sep 16 08:11:43 2024","interfaces":[{"interface":"back","average":{"1min":0.499,"5min":0.499,"15min":0.499},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":0.73399999999999999,"5min":0.73399999999999999,"15min":0.73399999999999999},"last":0.36599999999999999},{"interface":"front","average":{"1min":0.54000000000000004,"5min":0.54000000000000004,"15min":0.54000000000000004},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":0.78900000000000003,"5min":0.78900000000000003,"15min":0.78900000000000003},"last":0.60399999999999998}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.54300000000000004}]}]},{"osd":5,"up_from":39,"seq":167503724567,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5656,"kb_used_data":592,"kb_used_omap":0,"kb_used_meta":5056,"kb_avail":93739496,"statfs":{"total":95995035648,"available":95989243904,"internally_reserved":0,"allocated":606208,"data_stored":244910,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5177344},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Sep 16 08:11:16 2024","interfaces":[{"interface":"back","average":{"1min":0.54800000000000004,"5min":0.54800000000000004,"15min":0.54800000000000004},"min":{"1min":0.41699999999999998,"5min":0.41699999999999998,"15min":0.41699999999999998},"max":{"1min":0.82199999999999995,"5min":0.82199999999999995,"15min":0.82199999999999995},"last":0.39500000000000002},{"interface":"front","average":{"1min":0.64900000000000002,"5min":0.64900000000000002,"15min":0.64900000000000002},"min":{"1min":0.39200000000000002,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":2.1920000000000002,"5min":2.1920000000000002,"15min":2.1920000000000002},"last":0.621}]},{"osd":1,"last update":"Mon Sep 16 08:11:16 2024","interfaces":[{"interface":"back","average":{"1min":0.57899999999999996,"5min":0.57899999999999996,"15min":0.57899999999999996},"min":{"1min":0.442,"5min":0.442,"15min":0.442},"max":{"1min":0.88900000000000001,"5min":0.88900000000000001,"15min":0.88900000000000001},"last":0.437},{"interface":"front","average":{"1min":0.53500000000000003,"5min":0.53500000000000003,"15min":0.53500000000000003},"min":{"1min":0.40600000000000003,"5min":0.40600000000000003,"15min":0.40600000000000003},"max":{"1min":0.70099999999999996,"5min":0.70099999999999996,"15min":0.70099999999999996},"last":0.59099999999999997}]},{"osd":2,"last update":"Mon Sep 16 08:11:16 2024","interfaces":[{"interface":"back","average":{"1min":0.52700000000000002,"5min":0.52700000000000002,"15min":0.52700000000000002},"min":{"1min":0.39500000000000002,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":0.73199999999999998,"5min":0.73199999999999998,"15min":0.73199999999999998},"last":0.53200000000000003},{"interface":"front","average":{"1min":0.56699999999999995,"5min":0.56699999999999995,"15min":0.56699999999999995},"min":{"1min":0.41499999999999998,"5min":0.41499999999999998,"15min":0.41499999999999998},"max":{"1min":0.76700000000000002,"5min":0.76700000000000002,"15min":0.76700000000000002},"last":0.504}]},{"osd":3,"last update":"Mon Sep 16 08:11:16 2024","interfaces":[{"interface":"back","average":{"1min":0.54800000000000004,"5min":0.54800000000000004,"15min":0.54800000000000004},"min":{"1min":0.375,"5min":0.375,"15min":0.375},"max":{"1min":0.70899999999999996,"5min":0.70899999999999996,"15min":0.70899999999999996},"last":0.49099999999999999},{"interface":"front","average":{"1min":0.65800000000000003,"5min":0.65800000000000003,"15min":0.65800000000000003},"min":{"1min":0.45800000000000002,"5min":0.45800000000000002,"15min":0.45800000000000002},"max":{"1min":2.2090000000000001,"5min":2.2090000000000001,"15min":2.2090000000000001},"last":0.57799999999999996}]},{"osd":4,"last update":"Mon Sep 16 08:11:16 2024","interfaces":[{"interface":"back","average":{"1min":0.34899999999999998,"5min":0.34899999999999998,"15min":0.34899999999999998},"min":{"1min":0.25700000000000001,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":0.49299999999999999,"5min":0.49299999999999999,"15min":0.49299999999999999},"last":0.44600000000000001},{"interface":"front","average":{"1min":0.373,"5min":0.373,"15min":0.373},"min":{"1min":0.26300000000000001,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":0.59999999999999998,"5min":0.59999999999999998,"15min":0.59999999999999998},"last":0.40799999999999997}]},{"osd":6,"last update":"Mon Sep 16 08:11:44 2024","interfaces":[{"interface":"back","average":{"1min":0.47299999999999998,"5min":0.47299999999999998,"15min":0.47299999999999998},"min":{"1min":0.29499999999999998,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":0.76600000000000001,"5min":0.76600000000000001,"15min":0.76600000000000001},"last":0.56999999999999995},{"interface":"front","average":{"1min":0.49199999999999999,"5min":0.49199999999999999,"15min":0.49199999999999999},"min":{"1min":0.30099999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":0.77900000000000003,"5min":0.77900000000000003,"15min":0.77900000000000003},"last":0.52100000000000002}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.54900000000000004}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-16T08:12:00.833 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 5622b2fe-7402-11ef-bceb-c7b262605968 -- ceph pg dump --format=json 2024-09-16T08:12:01.059 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/5622b2fe-7402-11ef-bceb-c7b262605968/mon.smithi028/config 2024-09-16T08:12:01.430 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:12:01 smithi028 ceph-mon[23744]: pgmap v195: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-16T08:12:01.430 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:12:01 smithi028 ceph-mon[23744]: from='client.14440 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-16T08:12:01.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:12:01 smithi144 ceph-mon[25668]: pgmap v195: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-16T08:12:01.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:12:01 smithi144 ceph-mon[25668]: from='client.14440 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-16T08:12:02.769 INFO:teuthology.orchestra.run.smithi028.stdout: 2024-09-16T08:12:02.770 INFO:teuthology.orchestra.run.smithi028.stderr:dumped all 2024-09-16T08:12:03.387 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:12:03 smithi028 ceph-mon[23744]: pgmap v196: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-16T08:12:03.413 INFO:teuthology.orchestra.run.smithi028.stdout:{"pg_ready":true,"pg_map":{"version":196,"stamp":"2024-09-16T08:12:01.644486+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":117,"ondisk_log_size":117,"up":3,"acting":3,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":3,"num_osds":8,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":749961216,"kb_used":48812,"kb_used_data":6316,"kb_used_omap":0,"kb_used_meta":42432,"kb_avail":749912404,"statfs":{"total":767960285184,"available":767910301696,"internally_reserved":0,"allocated":6467584,"data_stored":3547795,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":43450368},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002578"},"pg_stats":[{"pgid":"1.0","version":"22'117","reported_seq":226,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-16T08:11:09.082247+0000","last_change":"2024-09-16T08:11:09.082247+0000","last_active":"2024-09-16T08:11:09.082247+0000","last_peered":"2024-09-16T08:11:09.082247+0000","last_clean":"2024-09-16T08:11:09.082247+0000","last_became_active":"2024-09-16T08:11:09.081812+0000","last_became_peered":"2024-09-16T08:11:09.081812+0000","last_unstale":"2024-09-16T08:11:09.082247+0000","last_undegraded":"2024-09-16T08:11:09.082247+0000","last_fullsized":"2024-09-16T08:11:09.082247+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-16T08:09:06.094292+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-16T08:09:06.094292+0000","last_clean_scrub_stamp":"2024-09-16T08:09:06.094292+0000","objects_scrubbed":0,"log_size":117,"ondisk_log_size":117,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-17T09:29:48.315968+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]}],"pool_stats":[{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":3727360,"data_stored":3702496,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":117,"ondisk_log_size":117,"up":3,"acting":3,"num_store_stats":7}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201294,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6052,"kb_used_data":1116,"kb_used_omap":0,"kb_used_meta":4928,"kb_avail":93739100,"statfs":{"total":95995035648,"available":95988838400,"internally_reserved":0,"allocated":1142784,"data_stored":774161,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5046272},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.51300000000000001}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.60799999999999998}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.45800000000000002}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.63800000000000001}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.58999999999999997}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.66400000000000003}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.71499999999999997}]}]},{"osd":6,"up_from":47,"seq":201863462931,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6116,"kb_used_data":1116,"kb_used_omap":0,"kb_used_meta":4992,"kb_avail":93739036,"statfs":{"total":95995035648,"available":95988772864,"internally_reserved":0,"allocated":1142784,"data_stored":774161,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5111808},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Sep 16 08:11:40 2024","interfaces":[{"interface":"back","average":{"1min":0.51800000000000002,"5min":0.51800000000000002,"15min":0.51800000000000002},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":0.72299999999999998,"5min":0.72299999999999998,"15min":0.72299999999999998},"last":0.60599999999999998},{"interface":"front","average":{"1min":0.53600000000000003,"5min":0.53600000000000003,"15min":0.53600000000000003},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":0.68200000000000005,"5min":0.68200000000000005,"15min":0.68200000000000005},"last":0.64700000000000002}]},{"osd":1,"last update":"Mon Sep 16 08:11:40 2024","interfaces":[{"interface":"back","average":{"1min":0.55400000000000005,"5min":0.55400000000000005,"15min":0.55400000000000005},"min":{"1min":0.373,"5min":0.373,"15min":0.373},"max":{"1min":0.746,"5min":0.746,"15min":0.746},"last":0.55300000000000005},{"interface":"front","average":{"1min":0.54400000000000004,"5min":0.54400000000000004,"15min":0.54400000000000004},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":0.83199999999999996,"5min":0.83199999999999996,"15min":0.83199999999999996},"last":0.56999999999999995}]},{"osd":2,"last update":"Mon Sep 16 08:11:40 2024","interfaces":[{"interface":"back","average":{"1min":0.54500000000000004,"5min":0.54500000000000004,"15min":0.54500000000000004},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":0.78100000000000003,"5min":0.78100000000000003,"15min":0.78100000000000003},"last":0.51300000000000001},{"interface":"front","average":{"1min":0.57199999999999995,"5min":0.57199999999999995,"15min":0.57199999999999995},"min":{"1min":0.41099999999999998,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":0.755,"5min":0.755,"15min":0.755},"last":0.58299999999999996}]},{"osd":3,"last update":"Mon Sep 16 08:11:40 2024","interfaces":[{"interface":"back","average":{"1min":0.55300000000000005,"5min":0.55300000000000005,"15min":0.55300000000000005},"min":{"1min":0.40600000000000003,"5min":0.40600000000000003,"15min":0.40600000000000003},"max":{"1min":0.68500000000000005,"5min":0.68500000000000005,"15min":0.68500000000000005},"last":0.63},{"interface":"front","average":{"1min":0.56699999999999995,"5min":0.56699999999999995,"15min":0.56699999999999995},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":0.80800000000000005,"5min":0.80800000000000005,"15min":0.80800000000000005},"last":0.623}]},{"osd":4,"last update":"Mon Sep 16 08:11:40 2024","interfaces":[{"interface":"back","average":{"1min":0.47499999999999998,"5min":0.47499999999999998,"15min":0.47499999999999998},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":0.74099999999999999,"5min":0.74099999999999999,"15min":0.74099999999999999},"last":0.61499999999999999},{"interface":"front","average":{"1min":0.45800000000000002,"5min":0.45800000000000002,"15min":0.45800000000000002},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":0.75700000000000001,"5min":0.75700000000000001,"15min":0.75700000000000001},"last":0.59799999999999998}]},{"osd":5,"last update":"Mon Sep 16 08:11:40 2024","interfaces":[{"interface":"back","average":{"1min":0.48599999999999999,"5min":0.48599999999999999,"15min":0.48599999999999999},"min":{"1min":0.311,"5min":0.311,"15min":0.311},"max":{"1min":0.81599999999999995,"5min":0.81599999999999995,"15min":0.81599999999999995},"last":0.53300000000000003},{"interface":"front","average":{"1min":0.505,"5min":0.505,"15min":0.505},"min":{"1min":0.317,"5min":0.317,"15min":0.317},"max":{"1min":0.751,"5min":0.751,"15min":0.751},"last":0.70899999999999996}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.67300000000000004}]}]},{"osd":1,"up_from":14,"seq":60129542188,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":6360,"kb_used_data":592,"kb_used_omap":0,"kb_used_meta":5760,"kb_avail":93738792,"statfs":{"total":95995035648,"available":95988523008,"internally_reserved":0,"allocated":606208,"data_stored":244968,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5898240},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Sep 16 08:11:46 2024","interfaces":[{"interface":"back","average":{"1min":0.39600000000000002,"5min":0.36099999999999999,"15min":0.33700000000000002},"min":{"1min":0.28599999999999998,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":0.57699999999999996,"5min":0.63500000000000001,"15min":0.63500000000000001},"last":0.35099999999999998},{"interface":"front","average":{"1min":0.43099999999999999,"5min":0.379,"15min":0.35499999999999998},"min":{"1min":0.27700000000000002,"5min":0.214,"15min":0.214},"max":{"1min":0.64200000000000002,"5min":0.64200000000000002,"15min":0.64200000000000002},"last":0.50700000000000001}]},{"osd":2,"last update":"Mon Sep 16 08:11:11 2024","interfaces":[{"interface":"back","average":{"1min":0.46400000000000002,"5min":0.41099999999999998,"15min":0.40200000000000002},"min":{"1min":0.318,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":0.65700000000000003,"5min":0.65700000000000003,"15min":0.65700000000000003},"last":0.44600000000000001},{"interface":"front","average":{"1min":0.42799999999999999,"5min":0.41199999999999998,"15min":0.40899999999999997},"min":{"1min":0.33700000000000002,"5min":0.31,"15min":0.31},"max":{"1min":0.59299999999999997,"5min":0.61699999999999999,"15min":0.61699999999999999},"last":0.41299999999999998}]},{"osd":3,"last update":"Mon Sep 16 08:11:36 2024","interfaces":[{"interface":"back","average":{"1min":0.433,"5min":0.45900000000000002,"15min":0.46300000000000002},"min":{"1min":0.314,"5min":0.314,"15min":0.314},"max":{"1min":0.60299999999999998,"5min":0.60299999999999998,"15min":0.60299999999999998},"last":0.46400000000000002},{"interface":"front","average":{"1min":0.48199999999999998,"5min":0.49199999999999999,"15min":0.49399999999999999},"min":{"1min":0.36699999999999999,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":0.68999999999999995,"5min":0.69999999999999996,"15min":0.69999999999999996},"last":0.49199999999999999}]},{"osd":4,"last update":"Mon Sep 16 08:10:53 2024","interfaces":[{"interface":"back","average":{"1min":0.63700000000000001,"5min":0.63700000000000001,"15min":0.63700000000000001},"min":{"1min":0.47099999999999997,"5min":0.47099999999999997,"15min":0.47099999999999997},"max":{"1min":0.81200000000000006,"5min":0.81200000000000006,"15min":0.81200000000000006},"last":0.66000000000000003},{"interface":"front","average":{"1min":0.63300000000000001,"5min":0.63300000000000001,"15min":0.63300000000000001},"min":{"1min":0.48199999999999998,"5min":0.48199999999999998,"15min":0.48199999999999998},"max":{"1min":0.84399999999999997,"5min":0.84399999999999997,"15min":0.84399999999999997},"last":0.66500000000000004}]},{"osd":5,"last update":"Mon Sep 16 08:11:21 2024","interfaces":[{"interface":"back","average":{"1min":0.61799999999999999,"5min":0.61799999999999999,"15min":0.61799999999999999},"min":{"1min":0.40300000000000002,"5min":0.40300000000000002,"15min":0.40300000000000002},"max":{"1min":0.85699999999999998,"5min":0.85699999999999998,"15min":0.85699999999999998},"last":0.56899999999999995},{"interface":"front","average":{"1min":0.64200000000000002,"5min":0.64200000000000002,"15min":0.64200000000000002},"min":{"1min":0.433,"5min":0.433,"15min":0.433},"max":{"1min":0.80100000000000005,"5min":0.80100000000000005,"15min":0.80100000000000005},"last":0.60899999999999999}]},{"osd":6,"last update":"Mon Sep 16 08:11:44 2024","interfaces":[{"interface":"back","average":{"1min":0.66400000000000003,"5min":0.66400000000000003,"15min":0.66400000000000003},"min":{"1min":0.40100000000000002,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":0.98999999999999999,"5min":0.98999999999999999,"15min":0.98999999999999999},"last":0.55600000000000005},{"interface":"front","average":{"1min":0.65000000000000002,"5min":0.65000000000000002,"15min":0.65000000000000002},"min":{"1min":0.46600000000000003,"5min":0.46600000000000003,"15min":0.46600000000000003},"max":{"1min":0.88600000000000001,"5min":0.88600000000000001,"15min":0.88600000000000001},"last":0.59799999999999998}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.57999999999999996}]}]},{"osd":0,"up_from":9,"seq":38654705713,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6884,"kb_used_data":1116,"kb_used_omap":0,"kb_used_meta":5760,"kb_avail":93738268,"statfs":{"total":95995035648,"available":95987986432,"internally_reserved":0,"allocated":1142784,"data_stored":774161,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5898240},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Sep 16 08:11:46 2024","interfaces":[{"interface":"back","average":{"1min":0.35299999999999998,"5min":0.33400000000000002,"15min":0.316},"min":{"1min":0.23100000000000001,"5min":0.19500000000000001,"15min":0.19500000000000001},"max":{"1min":0.53900000000000003,"5min":0.55100000000000005,"15min":0.55100000000000005},"last":0.33500000000000002},{"interface":"front","average":{"1min":0.41099999999999998,"5min":0.36699999999999999,"15min":0.34100000000000003},"min":{"1min":0.26500000000000001,"5min":0.187,"15min":0.187},"max":{"1min":0.53800000000000003,"5min":0.56899999999999995,"15min":0.56899999999999995},"last":0.42199999999999999}]},{"osd":2,"last update":"Mon Sep 16 08:11:08 2024","interfaces":[{"interface":"back","average":{"1min":0.42599999999999999,"5min":0.42499999999999999,"15min":0.42499999999999999},"min":{"1min":0.24099999999999999,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":0.63300000000000001,"5min":0.63300000000000001,"15min":0.63300000000000001},"last":0.47699999999999998},{"interface":"front","average":{"1min":0.45400000000000001,"5min":0.442,"15min":0.44},"min":{"1min":0.28399999999999997,"5min":0.224,"15min":0.224},"max":{"1min":0.67400000000000004,"5min":0.68999999999999995,"15min":0.68999999999999995},"last":0.34999999999999998}]},{"osd":3,"last update":"Mon Sep 16 08:11:34 2024","interfaces":[{"interface":"back","average":{"1min":0.44500000000000001,"5min":0.48499999999999999,"15min":0.49099999999999999},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":0.59099999999999997,"5min":0.72799999999999998,"15min":0.72799999999999998},"last":0.45600000000000002},{"interface":"front","average":{"1min":0.45700000000000002,"5min":0.47999999999999998,"15min":0.48399999999999999},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":0.63900000000000001,"5min":0.64500000000000002,"15min":0.64500000000000002},"last":0.44700000000000001}]},{"osd":4,"last update":"Mon Sep 16 08:10:56 2024","interfaces":[{"interface":"back","average":{"1min":0.61199999999999999,"5min":0.61199999999999999,"15min":0.61199999999999999},"min":{"1min":0.40600000000000003,"5min":0.40600000000000003,"15min":0.40600000000000003},"max":{"1min":0.79900000000000004,"5min":0.79900000000000004,"15min":0.79900000000000004},"last":0.70099999999999996},{"interface":"front","average":{"1min":0.626,"5min":0.626,"15min":0.626},"min":{"1min":0.44400000000000001,"5min":0.44400000000000001,"15min":0.44400000000000001},"max":{"1min":0.90300000000000002,"5min":0.90300000000000002,"15min":0.90300000000000002},"last":0.71199999999999997}]},{"osd":5,"last update":"Mon Sep 16 08:11:16 2024","interfaces":[{"interface":"back","average":{"1min":0.63200000000000001,"5min":0.63200000000000001,"15min":0.63200000000000001},"min":{"1min":0.46600000000000003,"5min":0.46600000000000003,"15min":0.46600000000000003},"max":{"1min":0.81100000000000005,"5min":0.81100000000000005,"15min":0.81100000000000005},"last":0.67200000000000004},{"interface":"front","average":{"1min":0.63900000000000001,"5min":0.63900000000000001,"15min":0.63900000000000001},"min":{"1min":0.52100000000000002,"5min":0.52100000000000002,"15min":0.52100000000000002},"max":{"1min":0.82299999999999995,"5min":0.82299999999999995,"15min":0.82299999999999995},"last":0.621}]},{"osd":6,"last update":"Mon Sep 16 08:11:40 2024","interfaces":[{"interface":"back","average":{"1min":0.61399999999999999,"5min":0.61399999999999999,"15min":0.61399999999999999},"min":{"1min":0.434,"5min":0.434,"15min":0.434},"max":{"1min":0.79500000000000004,"5min":0.79500000000000004,"15min":0.79500000000000004},"last":0.72599999999999998},{"interface":"front","average":{"1min":0.64400000000000002,"5min":0.64400000000000002,"15min":0.64400000000000002},"min":{"1min":0.45600000000000002,"5min":0.45600000000000002,"15min":0.45600000000000002},"max":{"1min":0.90500000000000003,"5min":0.90500000000000003,"15min":0.90500000000000003},"last":0.64400000000000002}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.63100000000000001}]}]},{"osd":2,"up_from":19,"seq":81604378662,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":6300,"kb_used_data":596,"kb_used_omap":0,"kb_used_meta":5696,"kb_avail":93738852,"statfs":{"total":95995035648,"available":95988584448,"internally_reserved":0,"allocated":610304,"data_stored":245233,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5832704},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Sep 16 08:11:08 2024","interfaces":[{"interface":"back","average":{"1min":0.43099999999999999,"5min":0.378,"15min":0.36899999999999999},"min":{"1min":0.314,"5min":0.221,"15min":0.221},"max":{"1min":0.67900000000000005,"5min":0.67900000000000005,"15min":0.67900000000000005},"last":0.47899999999999998},{"interface":"front","average":{"1min":0.44500000000000001,"5min":0.378,"15min":0.36699999999999999},"min":{"1min":0.29099999999999998,"5min":0.245,"15min":0.245},"max":{"1min":0.59199999999999997,"5min":0.59199999999999997,"15min":0.59199999999999997},"last":0.56499999999999995}]},{"osd":1,"last update":"Mon Sep 16 08:11:08 2024","interfaces":[{"interface":"back","average":{"1min":0.45700000000000002,"5min":0.38900000000000001,"15min":0.377},"min":{"1min":0.33200000000000002,"5min":0.252,"15min":0.252},"max":{"1min":0.56799999999999995,"5min":0.72999999999999998,"15min":0.72999999999999998},"last":0.441},{"interface":"front","average":{"1min":0.45700000000000002,"5min":0.377,"15min":0.36399999999999999},"min":{"1min":0.30299999999999999,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":0.70399999999999996,"5min":0.70399999999999996,"15min":0.70399999999999996},"last":0.52900000000000003}]},{"osd":3,"last update":"Mon Sep 16 08:11:31 2024","interfaces":[{"interface":"back","average":{"1min":0.39600000000000002,"5min":0.45000000000000001,"15min":0.45900000000000002},"min":{"1min":0.29099999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":0.60499999999999998,"5min":0.64900000000000002,"15min":0.64900000000000002},"last":0.51100000000000001},{"interface":"front","average":{"1min":0.47499999999999998,"5min":0.504,"15min":0.50900000000000001},"min":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":0.64100000000000001,"5min":0.71199999999999997,"15min":0.71199999999999997},"last":0.57699999999999996}]},{"osd":4,"last update":"Mon Sep 16 08:11:56 2024","interfaces":[{"interface":"back","average":{"1min":0.60399999999999998,"5min":0.63900000000000001,"15min":0.64500000000000002},"min":{"1min":0.44700000000000001,"5min":0.44700000000000001,"15min":0.44700000000000001},"max":{"1min":0.88400000000000001,"5min":0.88400000000000001,"15min":0.88400000000000001},"last":0.69799999999999995},{"interface":"front","average":{"1min":0.63300000000000001,"5min":0.63700000000000001,"15min":0.63700000000000001},"min":{"1min":0.47399999999999998,"5min":0.46800000000000003,"15min":0.46800000000000003},"max":{"1min":0.90200000000000002,"5min":0.90200000000000002,"15min":0.90200000000000002},"last":0.78700000000000003}]},{"osd":5,"last update":"Mon Sep 16 08:11:17 2024","interfaces":[{"interface":"back","average":{"1min":0.59399999999999997,"5min":0.59399999999999997,"15min":0.59399999999999997},"min":{"1min":0.39900000000000002,"5min":0.39900000000000002,"15min":0.39900000000000002},"max":{"1min":0.876,"5min":0.876,"15min":0.876},"last":0.747},{"interface":"front","average":{"1min":0.60699999999999998,"5min":0.60699999999999998,"15min":0.60699999999999998},"min":{"1min":0.44400000000000001,"5min":0.44400000000000001,"15min":0.44400000000000001},"max":{"1min":0.81699999999999995,"5min":0.81699999999999995,"15min":0.81699999999999995},"last":0.76400000000000001}]},{"osd":6,"last update":"Mon Sep 16 08:11:42 2024","interfaces":[{"interface":"back","average":{"1min":0.621,"5min":0.621,"15min":0.621},"min":{"1min":0.46700000000000003,"5min":0.46700000000000003,"15min":0.46700000000000003},"max":{"1min":0.83299999999999996,"5min":0.83299999999999996,"15min":0.83299999999999996},"last":0.77400000000000002},{"interface":"front","average":{"1min":0.61899999999999999,"5min":0.61899999999999999,"15min":0.61899999999999999},"min":{"1min":0.433,"5min":0.433,"15min":0.433},"max":{"1min":0.84099999999999997,"5min":0.84099999999999997,"15min":0.84099999999999997},"last":0.68100000000000005}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.80200000000000005}]}]},{"osd":3,"up_from":26,"seq":111669149730,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5720,"kb_used_data":592,"kb_used_omap":0,"kb_used_meta":5120,"kb_avail":93739432,"statfs":{"total":95995035648,"available":95989178368,"internally_reserved":0,"allocated":606208,"data_stored":244968,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5242880},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Sep 16 08:11:32 2024","interfaces":[{"interface":"back","average":{"1min":0.41699999999999998,"5min":0.40000000000000002,"15min":0.39700000000000002},"min":{"1min":0.253,"5min":0.253,"15min":0.253},"max":{"1min":1.0980000000000001,"5min":1.0980000000000001,"15min":1.0980000000000001},"last":0.36399999999999999},{"interface":"front","average":{"1min":0.42199999999999999,"5min":0.39200000000000002,"15min":0.38700000000000001},"min":{"1min":0.20399999999999999,"5min":0.20399999999999999,"15min":0.20399999999999999},"max":{"1min":0.61699999999999999,"5min":0.61699999999999999,"15min":0.61699999999999999},"last":0.41699999999999998}]},{"osd":1,"last update":"Mon Sep 16 08:11:32 2024","interfaces":[{"interface":"back","average":{"1min":0.41599999999999998,"5min":0.41199999999999998,"15min":0.41099999999999998},"min":{"1min":0.25800000000000001,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":0.58399999999999996,"5min":0.58399999999999996,"15min":0.58399999999999996},"last":0.376},{"interface":"front","average":{"1min":0.40899999999999997,"5min":0.39700000000000002,"15min":0.39500000000000002},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":0.70999999999999996,"5min":0.70999999999999996,"15min":0.70999999999999996},"last":0.39100000000000001}]},{"osd":2,"last update":"Mon Sep 16 08:11:32 2024","interfaces":[{"interface":"back","average":{"1min":0.47799999999999998,"5min":0.48599999999999999,"15min":0.48799999999999999},"min":{"1min":0.34999999999999998,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":0.69399999999999995,"5min":0.70299999999999996,"15min":0.70299999999999996},"last":0.496},{"interface":"front","average":{"1min":0.42999999999999999,"5min":0.46600000000000003,"15min":0.47199999999999998},"min":{"1min":0.217,"5min":0.217,"15min":0.217},"max":{"1min":0.66000000000000003,"5min":0.66000000000000003,"15min":0.66000000000000003},"last":0.48799999999999999}]},{"osd":4,"last update":"Mon Sep 16 08:11:52 2024","interfaces":[{"interface":"back","average":{"1min":0.59399999999999997,"5min":0.56699999999999995,"15min":0.56299999999999994},"min":{"1min":0.42099999999999999,"5min":0.42099999999999999,"15min":0.42099999999999999},"max":{"1min":0.98999999999999999,"5min":0.98999999999999999,"15min":0.98999999999999999},"last":0.73799999999999999},{"interface":"front","average":{"1min":0.60599999999999998,"5min":0.59199999999999997,"15min":0.58999999999999997},"min":{"1min":0.46999999999999997,"5min":0.443,"15min":0.443},"max":{"1min":0.85099999999999998,"5min":0.86099999999999999,"15min":0.86099999999999999},"last":0.63600000000000001}]},{"osd":5,"last update":"Mon Sep 16 08:11:19 2024","interfaces":[{"interface":"back","average":{"1min":0.58399999999999996,"5min":0.58399999999999996,"15min":0.58399999999999996},"min":{"1min":0.40999999999999998,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":0.85299999999999998,"5min":0.85299999999999998,"15min":0.85299999999999998},"last":0.68200000000000005},{"interface":"front","average":{"1min":0.59399999999999997,"5min":0.59399999999999997,"15min":0.59399999999999997},"min":{"1min":0.438,"5min":0.438,"15min":0.438},"max":{"1min":0.79900000000000004,"5min":0.79900000000000004,"15min":0.79900000000000004},"last":0.59199999999999997}]},{"osd":6,"last update":"Mon Sep 16 08:11:45 2024","interfaces":[{"interface":"back","average":{"1min":0.63300000000000001,"5min":0.63300000000000001,"15min":0.63300000000000001},"min":{"1min":0.42499999999999999,"5min":0.42499999999999999,"15min":0.42499999999999999},"max":{"1min":0.90100000000000002,"5min":0.90100000000000002,"15min":0.90100000000000002},"last":0.66000000000000003},{"interface":"front","average":{"1min":0.63,"5min":0.63,"15min":0.63},"min":{"1min":0.46400000000000002,"5min":0.46400000000000002,"15min":0.46400000000000002},"max":{"1min":0.88500000000000001,"5min":0.88500000000000001,"15min":0.88500000000000001},"last":0.60999999999999999}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.56899999999999995}]}]},{"osd":4,"up_from":31,"seq":133143986204,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5724,"kb_used_data":596,"kb_used_omap":0,"kb_used_meta":5120,"kb_avail":93739428,"statfs":{"total":95995035648,"available":95989174272,"internally_reserved":0,"allocated":610304,"data_stored":245233,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5242880},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Sep 16 08:11:55 2024","interfaces":[{"interface":"back","average":{"1min":0.57299999999999995,"5min":0.56899999999999995,"15min":0.56799999999999995},"min":{"1min":0.33900000000000002,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":0.82599999999999996,"5min":0.82599999999999996,"15min":0.82599999999999996},"last":0.55600000000000005},{"interface":"front","average":{"1min":0.55300000000000005,"5min":0.57399999999999995,"15min":0.57799999999999996},"min":{"1min":0.39000000000000001,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":0.70999999999999996,"5min":0.84799999999999998,"15min":0.84799999999999998},"last":0.53500000000000003}]},{"osd":1,"last update":"Mon Sep 16 08:11:55 2024","interfaces":[{"interface":"back","average":{"1min":0.53500000000000003,"5min":0.56799999999999995,"15min":0.57399999999999995},"min":{"1min":0.34899999999999998,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":0.71299999999999997,"5min":0.80800000000000005,"15min":0.80800000000000005},"last":0.499},{"interface":"front","average":{"1min":0.57099999999999995,"5min":0.56299999999999994,"15min":0.56200000000000006},"min":{"1min":0.38200000000000001,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":0.71699999999999997,"5min":0.874,"15min":0.874},"last":0.59299999999999997}]},{"osd":2,"last update":"Mon Sep 16 08:11:55 2024","interfaces":[{"interface":"back","average":{"1min":0.59899999999999998,"5min":0.57699999999999996,"15min":0.57299999999999995},"min":{"1min":0.46899999999999997,"5min":0.318,"15min":0.318},"max":{"1min":0.746,"5min":0.81599999999999995,"15min":0.81599999999999995},"last":0.51900000000000002},{"interface":"front","average":{"1min":0.56999999999999995,"5min":0.57599999999999996,"15min":0.57699999999999996},"min":{"1min":0.44,"5min":0.379,"15min":0.379},"max":{"1min":0.754,"5min":0.94999999999999996,"15min":0.94999999999999996},"last":0.57799999999999996}]},{"osd":3,"last update":"Mon Sep 16 08:11:55 2024","interfaces":[{"interface":"back","average":{"1min":0.56100000000000005,"5min":0.58499999999999996,"15min":0.58899999999999997},"min":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":0.79100000000000004,"5min":0.91900000000000004,"15min":0.91900000000000004},"last":0.38500000000000001},{"interface":"front","average":{"1min":0.57299999999999995,"5min":0.56999999999999995,"15min":0.56999999999999995},"min":{"1min":0.39300000000000002,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":0.78100000000000003,"5min":0.84599999999999997,"15min":0.84599999999999997},"last":0.41499999999999998}]},{"osd":5,"last update":"Mon Sep 16 08:11:17 2024","interfaces":[{"interface":"back","average":{"1min":0.40300000000000002,"5min":0.40300000000000002,"15min":0.40300000000000002},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":0.69699999999999995,"5min":0.69699999999999995,"15min":0.69699999999999995},"last":0.46800000000000003},{"interface":"front","average":{"1min":0.41099999999999998,"5min":0.41099999999999998,"15min":0.41099999999999998},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":0.71699999999999997,"5min":0.71699999999999997,"15min":0.71699999999999997},"last":0.48299999999999998}]},{"osd":6,"last update":"Mon Sep 16 08:11:43 2024","interfaces":[{"interface":"back","average":{"1min":0.499,"5min":0.499,"15min":0.499},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":0.73399999999999999,"5min":0.73399999999999999,"15min":0.73399999999999999},"last":0.36599999999999999},{"interface":"front","average":{"1min":0.54000000000000004,"5min":0.54000000000000004,"15min":0.54000000000000004},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":0.78900000000000003,"5min":0.78900000000000003,"15min":0.78900000000000003},"last":0.60399999999999998}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.54300000000000004}]}]},{"osd":5,"up_from":39,"seq":167503724567,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5656,"kb_used_data":592,"kb_used_omap":0,"kb_used_meta":5056,"kb_avail":93739496,"statfs":{"total":95995035648,"available":95989243904,"internally_reserved":0,"allocated":606208,"data_stored":244910,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5177344},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Sep 16 08:11:16 2024","interfaces":[{"interface":"back","average":{"1min":0.54800000000000004,"5min":0.54800000000000004,"15min":0.54800000000000004},"min":{"1min":0.41699999999999998,"5min":0.41699999999999998,"15min":0.41699999999999998},"max":{"1min":0.82199999999999995,"5min":0.82199999999999995,"15min":0.82199999999999995},"last":0.39500000000000002},{"interface":"front","average":{"1min":0.64900000000000002,"5min":0.64900000000000002,"15min":0.64900000000000002},"min":{"1min":0.39200000000000002,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":2.1920000000000002,"5min":2.1920000000000002,"15min":2.1920000000000002},"last":0.621}]},{"osd":1,"last update":"Mon Sep 16 08:11:16 2024","interfaces":[{"interface":"back","average":{"1min":0.57899999999999996,"5min":0.57899999999999996,"15min":0.57899999999999996},"min":{"1min":0.442,"5min":0.442,"15min":0.442},"max":{"1min":0.88900000000000001,"5min":0.88900000000000001,"15min":0.88900000000000001},"last":0.437},{"interface":"front","average":{"1min":0.53500000000000003,"5min":0.53500000000000003,"15min":0.53500000000000003},"min":{"1min":0.40600000000000003,"5min":0.40600000000000003,"15min":0.40600000000000003},"max":{"1min":0.70099999999999996,"5min":0.70099999999999996,"15min":0.70099999999999996},"last":0.59099999999999997}]},{"osd":2,"last update":"Mon Sep 16 08:11:16 2024","interfaces":[{"interface":"back","average":{"1min":0.52700000000000002,"5min":0.52700000000000002,"15min":0.52700000000000002},"min":{"1min":0.39500000000000002,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":0.73199999999999998,"5min":0.73199999999999998,"15min":0.73199999999999998},"last":0.53200000000000003},{"interface":"front","average":{"1min":0.56699999999999995,"5min":0.56699999999999995,"15min":0.56699999999999995},"min":{"1min":0.41499999999999998,"5min":0.41499999999999998,"15min":0.41499999999999998},"max":{"1min":0.76700000000000002,"5min":0.76700000000000002,"15min":0.76700000000000002},"last":0.504}]},{"osd":3,"last update":"Mon Sep 16 08:11:16 2024","interfaces":[{"interface":"back","average":{"1min":0.54800000000000004,"5min":0.54800000000000004,"15min":0.54800000000000004},"min":{"1min":0.375,"5min":0.375,"15min":0.375},"max":{"1min":0.70899999999999996,"5min":0.70899999999999996,"15min":0.70899999999999996},"last":0.49099999999999999},{"interface":"front","average":{"1min":0.65800000000000003,"5min":0.65800000000000003,"15min":0.65800000000000003},"min":{"1min":0.45800000000000002,"5min":0.45800000000000002,"15min":0.45800000000000002},"max":{"1min":2.2090000000000001,"5min":2.2090000000000001,"15min":2.2090000000000001},"last":0.57799999999999996}]},{"osd":4,"last update":"Mon Sep 16 08:11:16 2024","interfaces":[{"interface":"back","average":{"1min":0.34899999999999998,"5min":0.34899999999999998,"15min":0.34899999999999998},"min":{"1min":0.25700000000000001,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":0.49299999999999999,"5min":0.49299999999999999,"15min":0.49299999999999999},"last":0.44600000000000001},{"interface":"front","average":{"1min":0.373,"5min":0.373,"15min":0.373},"min":{"1min":0.26300000000000001,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":0.59999999999999998,"5min":0.59999999999999998,"15min":0.59999999999999998},"last":0.40799999999999997}]},{"osd":6,"last update":"Mon Sep 16 08:11:44 2024","interfaces":[{"interface":"back","average":{"1min":0.47299999999999998,"5min":0.47299999999999998,"15min":0.47299999999999998},"min":{"1min":0.29499999999999998,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":0.76600000000000001,"5min":0.76600000000000001,"15min":0.76600000000000001},"last":0.56999999999999995},{"interface":"front","average":{"1min":0.49199999999999999,"5min":0.49199999999999999,"15min":0.49199999999999999},"min":{"1min":0.30099999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":0.77900000000000003,"5min":0.77900000000000003,"15min":0.77900000000000003},"last":0.52100000000000002}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.54900000000000004}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-16T08:12:03.415 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2024-09-16T08:12:03.415 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2024-09-16T08:12:03.415 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy 2024-09-16T08:12:03.415 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 5622b2fe-7402-11ef-bceb-c7b262605968 -- ceph health --format=json 2024-09-16T08:12:03.442 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:12:03 smithi144 ceph-mon[25668]: pgmap v196: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-16T08:12:03.636 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/5622b2fe-7402-11ef-bceb-c7b262605968/mon.smithi028/config 2024-09-16T08:12:04.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:12:04 smithi028 ceph-mon[23744]: from='client.14444 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-16T08:12:04.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:12:04 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi028.azvyzy/mirror_snapshot_schedule"}]: dispatch 2024-09-16T08:12:04.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:12:04 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi028.azvyzy/trash_purge_schedule"}]: dispatch 2024-09-16T08:12:04.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:12:04 smithi144 ceph-mon[25668]: from='client.14444 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-16T08:12:04.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:12:04 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi028.azvyzy/mirror_snapshot_schedule"}]: dispatch 2024-09-16T08:12:04.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:12:04 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi028.azvyzy/trash_purge_schedule"}]: dispatch 2024-09-16T08:12:05.427 INFO:teuthology.orchestra.run.smithi028.stdout: 2024-09-16T08:12:05.427 INFO:teuthology.orchestra.run.smithi028.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2024-09-16T08:12:05.427 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:12:05 smithi028 ceph-mon[23744]: pgmap v197: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-16T08:12:05.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:12:05 smithi144 ceph-mon[25668]: pgmap v197: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-16T08:12:06.017 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy done 2024-09-16T08:12:06.018 INFO:tasks.cephadm:Setup complete, yielding 2024-09-16T08:12:06.018 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-09-16T08:12:06.028 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi028.front.sepia.ceph.com 2024-09-16T08:12:06.028 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5622b2fe-7402-11ef-bceb-c7b262605968 -- bash -c 'ceph orch status' 2024-09-16T08:12:06.337 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:12:06 smithi028 ceph-mon[23744]: from='client.? 172.21.15.28:0/2851809376' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-09-16T08:12:06.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:12:06 smithi144 ceph-mon[25668]: from='client.? 172.21.15.28:0/2851809376' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-09-16T08:12:07.440 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:12:07 smithi028 ceph-mon[23744]: pgmap v198: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-16T08:12:07.444 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:12:07 smithi144 ceph-mon[25668]: pgmap v198: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-16T08:12:07.968 INFO:teuthology.orchestra.run.smithi028.stdout:Backend: cephadm 2024-09-16T08:12:07.968 INFO:teuthology.orchestra.run.smithi028.stdout:Available: Yes 2024-09-16T08:12:07.968 INFO:teuthology.orchestra.run.smithi028.stdout:Paused: No 2024-09-16T08:12:08.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:12:08 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-16T08:12:08.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:12:08 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T08:12:08.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:12:08 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-16T08:12:08.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:12:08 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' 2024-09-16T08:12:08.514 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5622b2fe-7402-11ef-bceb-c7b262605968 -- bash -c 'ceph orch ps' 2024-09-16T08:12:08.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:12:08 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-16T08:12:08.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:12:08 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T08:12:08.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:12:08 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-16T08:12:08.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:12:08 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' 2024-09-16T08:12:09.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:12:09 smithi028 ceph-mon[23744]: pgmap v199: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-16T08:12:09.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:12:09 smithi028 ceph-mon[23744]: from='client.14452 -' entity='client.admin' cmd=[{"prefix": "orch status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T08:12:09.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:12:09 smithi144 ceph-mon[25668]: pgmap v199: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-16T08:12:09.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:12:09 smithi144 ceph-mon[25668]: from='client.14452 -' entity='client.admin' cmd=[{"prefix": "orch status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T08:12:10.341 INFO:teuthology.orchestra.run.smithi028.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-16T08:12:10.341 INFO:teuthology.orchestra.run.smithi028.stdout:alertmanager.smithi028 smithi028 *:9093,9094 running (4m) 2m ago 5m 17.6M - ba2b418f427c 8e1f82e9293c 2024-09-16T08:12:10.341 INFO:teuthology.orchestra.run.smithi028.stdout:crash.smithi028 smithi028 running (5m) 2m ago 5m 6405k - 17.2.0 e1d6a67b021e 69e795c000fd 2024-09-16T08:12:10.341 INFO:teuthology.orchestra.run.smithi028.stdout:crash.smithi144 smithi144 running (4m) 66s ago 4m 6400k - 17.2.0 e1d6a67b021e bbe54b51649c 2024-09-16T08:12:10.342 INFO:teuthology.orchestra.run.smithi028.stdout:grafana.smithi028 smithi028 *:3000 running (3m) 2m ago 5m 42.9M - 8.3.5 dad864ee21e9 6ff027b7efc7 2024-09-16T08:12:10.342 INFO:teuthology.orchestra.run.smithi028.stdout:mgr.smithi028.azvyzy smithi028 *:9283 running (6m) 2m ago 6m 448M - 17.2.0 e1d6a67b021e 229e5e86f252 2024-09-16T08:12:10.342 INFO:teuthology.orchestra.run.smithi028.stdout:mgr.smithi144.utzszj smithi144 *:8443,9283 running (4m) 66s ago 4m 411M - 17.2.0 e1d6a67b021e ba5c5c8ef9ca 2024-09-16T08:12:10.342 INFO:teuthology.orchestra.run.smithi028.stdout:mon.smithi028 smithi028 running (6m) 2m ago 6m 44.1M 2048M 17.2.0 e1d6a67b021e a2f688697052 2024-09-16T08:12:10.342 INFO:teuthology.orchestra.run.smithi028.stdout:mon.smithi144 smithi144 running (4m) 66s ago 4m 41.8M 2048M 17.2.0 e1d6a67b021e 1c80e3ff3507 2024-09-16T08:12:10.342 INFO:teuthology.orchestra.run.smithi028.stdout:node-exporter.smithi028 smithi028 *:9100 running (5m) 2m ago 5m 16.9M - 1dbe0e931976 7ad46df29362 2024-09-16T08:12:10.342 INFO:teuthology.orchestra.run.smithi028.stdout:node-exporter.smithi144 smithi144 *:9100 running (4m) 66s ago 4m 18.6M - 1dbe0e931976 f85c6daf2fe0 2024-09-16T08:12:10.342 INFO:teuthology.orchestra.run.smithi028.stdout:osd.0 smithi028 running (4m) 2m ago 4m 34.0M 3217M 17.2.0 e1d6a67b021e 8184fbc3871f 2024-09-16T08:12:10.342 INFO:teuthology.orchestra.run.smithi028.stdout:osd.1 smithi028 running (3m) 2m ago 3m 35.3M 3217M 17.2.0 e1d6a67b021e 6911d5c68d54 2024-09-16T08:12:10.342 INFO:teuthology.orchestra.run.smithi028.stdout:osd.2 smithi028 running (3m) 2m ago 3m 30.7M 3217M 17.2.0 e1d6a67b021e 90b27d8375a2 2024-09-16T08:12:10.343 INFO:teuthology.orchestra.run.smithi028.stdout:osd.3 smithi028 running (2m) 2m ago 2m 28.0M 3217M 17.2.0 e1d6a67b021e dac477d8b04f 2024-09-16T08:12:10.343 INFO:teuthology.orchestra.run.smithi028.stdout:osd.4 smithi144 running (2m) 66s ago 2m 33.6M 3984M 17.2.0 e1d6a67b021e 86719c60975c 2024-09-16T08:12:10.343 INFO:teuthology.orchestra.run.smithi028.stdout:osd.5 smithi144 running (2m) 66s ago 119s 32.3M 3984M 17.2.0 e1d6a67b021e 70452c145c60 2024-09-16T08:12:10.343 INFO:teuthology.orchestra.run.smithi028.stdout:osd.6 smithi144 running (95s) 66s ago 95s 31.4M 3984M 17.2.0 e1d6a67b021e 7f9981be792f 2024-09-16T08:12:10.343 INFO:teuthology.orchestra.run.smithi028.stdout:osd.7 smithi144 running (71s) 66s ago 71s 13.0M 3984M 17.2.0 e1d6a67b021e dada74ddbe4c 2024-09-16T08:12:10.343 INFO:teuthology.orchestra.run.smithi028.stdout:prometheus.smithi028 smithi028 *:9095 running (4m) 2m ago 4m 56.5M - 514e6a882f6e fd3df637374d 2024-09-16T08:12:10.937 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5622b2fe-7402-11ef-bceb-c7b262605968 -- bash -c 'ceph orch ls' 2024-09-16T08:12:11.443 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:12:11 smithi028 ceph-mon[23744]: pgmap v200: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-16T08:12:11.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:12:11 smithi144 ceph-mon[25668]: pgmap v200: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-16T08:12:12.304 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:12:12 smithi028 ceph-mon[23744]: from='client.14456 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T08:12:12.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:12:12 smithi144 ceph-mon[25668]: from='client.14456 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T08:12:12.721 INFO:teuthology.orchestra.run.smithi028.stdout:NAME PORTS RUNNING REFRESHED AGE PLACEMENT 2024-09-16T08:12:12.722 INFO:teuthology.orchestra.run.smithi028.stdout:alertmanager ?:9093,9094 1/1 2m ago 6m count:1 2024-09-16T08:12:12.722 INFO:teuthology.orchestra.run.smithi028.stdout:crash 2/2 2m ago 6m * 2024-09-16T08:12:12.722 INFO:teuthology.orchestra.run.smithi028.stdout:grafana ?:3000 1/1 2m ago 6m count:1 2024-09-16T08:12:12.722 INFO:teuthology.orchestra.run.smithi028.stdout:mgr 2/2 2m ago 6m count:2 2024-09-16T08:12:12.722 INFO:teuthology.orchestra.run.smithi028.stdout:mon 2/2 2m ago 5m smithi028:172.21.15.28=smithi028;smithi144:172.21.15.144=smithi144;count:2 2024-09-16T08:12:12.722 INFO:teuthology.orchestra.run.smithi028.stdout:node-exporter ?:9100 2/2 2m ago 6m * 2024-09-16T08:12:12.722 INFO:teuthology.orchestra.run.smithi028.stdout:osd 8 2m ago - 2024-09-16T08:12:12.722 INFO:teuthology.orchestra.run.smithi028.stdout:prometheus ?:9095 1/1 2m ago 6m count:1 2024-09-16T08:12:13.283 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:12:13 smithi028 ceph-mon[23744]: pgmap v201: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-16T08:12:13.312 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5622b2fe-7402-11ef-bceb-c7b262605968 -- bash -c 'ceph orch host ls' 2024-09-16T08:12:13.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:12:13 smithi144 ceph-mon[25668]: pgmap v201: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-16T08:12:14.446 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:12:14 smithi028 ceph-mon[23744]: from='client.14460 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T08:12:14.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:12:14 smithi144 ceph-mon[25668]: from='client.14460 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T08:12:15.179 INFO:teuthology.orchestra.run.smithi028.stdout:HOST ADDR LABELS STATUS 2024-09-16T08:12:15.179 INFO:teuthology.orchestra.run.smithi028.stdout:smithi028 172.21.15.28 2024-09-16T08:12:15.179 INFO:teuthology.orchestra.run.smithi028.stdout:smithi144 172.21.15.144 2024-09-16T08:12:15.179 INFO:teuthology.orchestra.run.smithi028.stdout:2 hosts in cluster 2024-09-16T08:12:15.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:12:15 smithi028 ceph-mon[23744]: pgmap v202: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-16T08:12:15.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:12:15 smithi144 ceph-mon[25668]: pgmap v202: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-16T08:12:15.753 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5622b2fe-7402-11ef-bceb-c7b262605968 -- bash -c 'ceph orch device ls' 2024-09-16T08:12:16.389 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:12:16 smithi028 ceph-mon[23744]: from='client.14464 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T08:12:16.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:12:16 smithi144 ceph-mon[25668]: from='client.14464 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T08:12:17.262 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:12:17 smithi028 ceph-mon[23744]: pgmap v203: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-16T08:12:17.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:12:17 smithi144 ceph-mon[25668]: pgmap v203: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-16T08:12:17.713 INFO:teuthology.orchestra.run.smithi028.stdout:HOST PATH TYPE DEVICE ID SIZE AVAILABLE REJECT REASONS 2024-09-16T08:12:17.713 INFO:teuthology.orchestra.run.smithi028.stdout:smithi028 /dev/nvme0n1 ssd INTEL_SSDPED1D480GA_PHMB747401F4480DGN 480G LVM detected, locked 2024-09-16T08:12:17.713 INFO:teuthology.orchestra.run.smithi028.stdout:smithi144 /dev/nvme0n1 ssd INTEL_SSDPEDMD400G4_PHFT62050085400BGN 400G LVM detected, locked 2024-09-16T08:12:18.340 INFO:teuthology.run_tasks:Running task vip.exec... 2024-09-16T08:12:18.362 INFO:tasks.vip:Running commands on role host.a host ubuntu@smithi028.front.sepia.ceph.com 2024-09-16T08:12:18.363 DEBUG:teuthology.orchestra.run.smithi028:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c 'systemctl stop nfs-server' 2024-09-16T08:12:18.391 INFO:teuthology.orchestra.run.smithi028.stderr:+ systemctl stop nfs-server 2024-09-16T08:12:18.399 INFO:tasks.vip:Running commands on role host.b host ubuntu@smithi144.front.sepia.ceph.com 2024-09-16T08:12:18.399 DEBUG:teuthology.orchestra.run.smithi144:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c 'systemctl stop nfs-server' 2024-09-16T08:12:18.428 INFO:teuthology.orchestra.run.smithi144.stderr:+ systemctl stop nfs-server 2024-09-16T08:12:18.437 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-09-16T08:12:18.445 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi028.front.sepia.ceph.com 2024-09-16T08:12:18.446 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5622b2fe-7402-11ef-bceb-c7b262605968 -- bash -c 'ceph fs volume create foofs' 2024-09-16T08:12:19.446 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:12:19 smithi028 ceph-mon[23744]: pgmap v204: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-16T08:12:19.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:12:19 smithi028 ceph-mon[23744]: from='client.14468 -' entity='client.admin' cmd=[{"prefix": "orch device ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T08:12:19.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:12:19 smithi144 ceph-mon[25668]: pgmap v204: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-16T08:12:19.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:12:19 smithi144 ceph-mon[25668]: from='client.14468 -' entity='client.admin' cmd=[{"prefix": "orch device ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T08:12:21.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:12:21 smithi144 ceph-mon[25668]: pgmap v205: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-16T08:12:21.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:12:21 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foofs.meta"}]: dispatch 2024-09-16T08:12:21.697 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:12:21 smithi028 ceph-mon[23744]: pgmap v205: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-16T08:12:21.697 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:12:21 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foofs.meta"}]: dispatch 2024-09-16T08:12:22.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:12:22 smithi144 ceph-mon[25668]: from='client.14472 -' entity='client.admin' cmd=[{"prefix": "fs volume create", "name": "foofs", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T08:12:22.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:12:22 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foofs.meta"}]': finished 2024-09-16T08:12:22.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:12:22 smithi144 ceph-mon[25668]: osdmap e60: 8 total, 8 up, 8 in 2024-09-16T08:12:22.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:12:22 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foofs.data"}]: dispatch 2024-09-16T08:12:22.594 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:12:22 smithi028 ceph-mon[23744]: from='client.14472 -' entity='client.admin' cmd=[{"prefix": "fs volume create", "name": "foofs", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T08:12:22.594 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:12:22 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foofs.meta"}]': finished 2024-09-16T08:12:22.594 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:12:22 smithi028 ceph-mon[23744]: osdmap e60: 8 total, 8 up, 8 in 2024-09-16T08:12:22.595 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:12:22 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foofs.data"}]: dispatch 2024-09-16T08:12:22.595 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:12:22 smithi028 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-mon-smithi028[23740]: 2024-09-16T08:12:22.239+0000 7f23093cb700 -1 log_channel(cluster) log [ERR] : Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-09-16T08:12:22.871 INFO:teuthology.run_tasks:Running task cephadm.wait_for_service... 2024-09-16T08:12:22.886 INFO:tasks.cephadm:Waiting for ceph service mds.foofs to start (timeout 300)... 2024-09-16T08:12:22.887 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5622b2fe-7402-11ef-bceb-c7b262605968 -- ceph orch ls -f json 2024-09-16T08:12:23.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:12:23 smithi028 ceph-mon[23744]: pgmap v207: 33 pgs: 32 unknown, 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-16T08:12:23.448 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:12:23 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foofs.data"}]': finished 2024-09-16T08:12:23.448 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:12:23 smithi028 ceph-mon[23744]: osdmap e61: 8 total, 8 up, 8 in 2024-09-16T08:12:23.448 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:12:23 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "fs new", "fs_name": "foofs", "metadata": "cephfs.foofs.meta", "data": "cephfs.foofs.data"}]: dispatch 2024-09-16T08:12:23.448 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:12:23 smithi028 ceph-mon[23744]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-09-16T08:12:23.448 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:12:23 smithi028 ceph-mon[23744]: Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2024-09-16T08:12:23.448 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:12:23 smithi028 ceph-mon[23744]: osdmap e62: 8 total, 8 up, 8 in 2024-09-16T08:12:23.449 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:12:23 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd='[{"prefix": "fs new", "fs_name": "foofs", "metadata": "cephfs.foofs.meta", "data": "cephfs.foofs.data"}]': finished 2024-09-16T08:12:23.449 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:12:23 smithi028 ceph-mon[23744]: fsmap foofs:0 2024-09-16T08:12:23.449 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:12:23 smithi028 ceph-mon[23744]: Saving service mds.foofs spec with placement count:2 2024-09-16T08:12:23.449 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:12:23 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' 2024-09-16T08:12:23.449 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:12:23 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-16T08:12:23.450 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:12:23 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T08:12:23.450 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:12:23 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-16T08:12:23.450 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:12:23 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' 2024-09-16T08:12:23.450 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:12:23 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' 2024-09-16T08:12:23.450 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:12:23 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi028.srtgev", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-09-16T08:12:23.450 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:12:23 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi028.srtgev", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-09-16T08:12:23.451 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:12:23 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T08:12:23.451 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:12:23 smithi028 ceph-mon[23744]: Deploying daemon mds.foofs.smithi028.srtgev on smithi028 2024-09-16T08:12:23.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:12:23 smithi144 ceph-mon[25668]: pgmap v207: 33 pgs: 32 unknown, 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-16T08:12:23.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:12:23 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foofs.data"}]': finished 2024-09-16T08:12:23.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:12:23 smithi144 ceph-mon[25668]: osdmap e61: 8 total, 8 up, 8 in 2024-09-16T08:12:23.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:12:23 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "fs new", "fs_name": "foofs", "metadata": "cephfs.foofs.meta", "data": "cephfs.foofs.data"}]: dispatch 2024-09-16T08:12:23.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:12:23 smithi144 ceph-mon[25668]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-09-16T08:12:23.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:12:23 smithi144 ceph-mon[25668]: Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2024-09-16T08:12:23.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:12:23 smithi144 ceph-mon[25668]: osdmap e62: 8 total, 8 up, 8 in 2024-09-16T08:12:23.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:12:23 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd='[{"prefix": "fs new", "fs_name": "foofs", "metadata": "cephfs.foofs.meta", "data": "cephfs.foofs.data"}]': finished 2024-09-16T08:12:23.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:12:23 smithi144 ceph-mon[25668]: fsmap foofs:0 2024-09-16T08:12:23.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:12:23 smithi144 ceph-mon[25668]: Saving service mds.foofs spec with placement count:2 2024-09-16T08:12:23.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:12:23 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' 2024-09-16T08:12:23.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:12:23 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-16T08:12:23.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:12:23 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T08:12:23.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:12:23 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-16T08:12:23.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:12:23 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' 2024-09-16T08:12:23.583 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:12:23 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' 2024-09-16T08:12:23.583 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:12:23 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi028.srtgev", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-09-16T08:12:23.583 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:12:23 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi028.srtgev", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-09-16T08:12:23.583 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:12:23 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T08:12:23.583 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:12:23 smithi144 ceph-mon[25668]: Deploying daemon mds.foofs.smithi028.srtgev on smithi028 2024-09-16T08:12:24.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:12:24 smithi144 ceph-mon[25668]: osdmap e63: 8 total, 8 up, 8 in 2024-09-16T08:12:24.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:12:24 smithi144 ceph-mon[25668]: pgmap v211: 65 pgs: 14 creating+peering, 1 active+clean, 50 unknown; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-16T08:12:24.634 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:12:24 smithi028 ceph-mon[23744]: osdmap e63: 8 total, 8 up, 8 in 2024-09-16T08:12:24.634 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:12:24 smithi028 ceph-mon[23744]: pgmap v211: 65 pgs: 14 creating+peering, 1 active+clean, 50 unknown; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-16T08:12:25.446 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:12:25 smithi028 ceph-mon[23744]: osdmap e64: 8 total, 8 up, 8 in 2024-09-16T08:12:25.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:12:25 smithi144 ceph-mon[25668]: osdmap e64: 8 total, 8 up, 8 in 2024-09-16T08:12:25.661 INFO:teuthology.orchestra.run.smithi028.stdout: 2024-09-16T08:12:25.661 INFO:teuthology.orchestra.run.smithi028.stdout:[{"placement": {"count": 1}, "service_name": "alertmanager", "service_type": "alertmanager", "status": {"created": "2024-09-16T08:06:03.680757Z", "last_refresh": "2024-09-16T08:09:38.042267Z", "ports": [9093, 9094], "running": 1, "size": 1}}, {"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"created": "2024-09-16T08:05:58.113512Z", "last_refresh": "2024-09-16T08:09:38.042334Z", "running": 2, "size": 2}}, {"placement": {"count": 1}, "service_name": "grafana", "service_type": "grafana", "status": {"created": "2024-09-16T08:06:00.803468Z", "last_refresh": "2024-09-16T08:09:38.042399Z", "ports": [3000], "running": 1, "size": 1}}, {"events": ["2024-09-16T08:12:22.262443Z service:mds.foofs [INFO] \"service was created\""], "placement": {"count": 2}, "service_id": "foofs", "service_name": "mds.foofs", "service_type": "mds", "status": {"created": "2024-09-16T08:12:22.255272Z", "running": 0, "size": 2}}, {"placement": {"count": 2}, "service_name": "mgr", "service_type": "mgr", "status": {"created": "2024-09-16T08:05:56.955187Z", "last_refresh": "2024-09-16T08:09:38.042192Z", "running": 2, "size": 2}}, {"placement": {"count": 2, "hosts": ["smithi028:172.21.15.28=smithi028", "smithi144:172.21.15.144=smithi144"]}, "service_name": "mon", "service_type": "mon", "status": {"created": "2024-09-16T08:06:46.314972Z", "last_refresh": "2024-09-16T08:09:38.042045Z", "running": 2, "size": 2}}, {"placement": {"host_pattern": "*"}, "service_name": "node-exporter", "service_type": "node-exporter", "status": {"created": "2024-09-16T08:06:02.516246Z", "last_refresh": "2024-09-16T08:09:38.042463Z", "ports": [9100], "running": 2, "size": 2}}, {"service_name": "osd", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9", "container_image_name": "quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a", "last_refresh": "2024-09-16T08:09:38.042598Z", "running": 8, "size": 8}, "unmanaged": true}, {"placement": {"count": 1}, "service_name": "prometheus", "service_type": "prometheus", "status": {"created": "2024-09-16T08:05:59.457450Z", "last_refresh": "2024-09-16T08:09:38.042527Z", "ports": [9095], "running": 1, "size": 1}}] 2024-09-16T08:12:26.391 INFO:tasks.cephadm:mds.foofs has 0/2 2024-09-16T08:12:26.561 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:12:26 smithi028 ceph-mon[23744]: pgmap v213: 65 pgs: 24 active+clean, 14 creating+peering, 27 unknown; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-16T08:12:26.561 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:12:26 smithi028 ceph-mon[23744]: from='client.14476 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-16T08:12:26.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:12:26 smithi144 ceph-mon[25668]: pgmap v213: 65 pgs: 24 active+clean, 14 creating+peering, 27 unknown; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-16T08:12:26.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:12:26 smithi144 ceph-mon[25668]: from='client.14476 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-16T08:12:27.392 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5622b2fe-7402-11ef-bceb-c7b262605968 -- ceph orch ls -f json 2024-09-16T08:12:28.022 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:12:27 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' 2024-09-16T08:12:28.022 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:12:27 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi144.oozgsi", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-09-16T08:12:28.022 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:12:27 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi144.oozgsi", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-09-16T08:12:28.022 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:12:27 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T08:12:28.022 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:12:27 smithi028 ceph-mon[23744]: Deploying daemon mds.foofs.smithi144.oozgsi on smithi144 2024-09-16T08:12:28.022 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:12:27 smithi028 ceph-mon[23744]: mds.? [v2:172.21.15.28:6834/800761142,v1:172.21.15.28:6835/800761142] up:boot 2024-09-16T08:12:28.022 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:12:27 smithi028 ceph-mon[23744]: daemon mds.foofs.smithi028.srtgev assigned to filesystem foofs as rank 0 (now has 1 ranks) 2024-09-16T08:12:28.022 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:12:27 smithi028 ceph-mon[23744]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-09-16T08:12:28.022 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:12:27 smithi028 ceph-mon[23744]: Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2024-09-16T08:12:28.022 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:12:27 smithi028 ceph-mon[23744]: Cluster is now healthy 2024-09-16T08:12:28.023 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:12:27 smithi028 ceph-mon[23744]: fsmap foofs:0 1 up:standby 2024-09-16T08:12:28.023 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:12:27 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi028.srtgev"}]: dispatch 2024-09-16T08:12:28.023 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:12:27 smithi028 ceph-mon[23744]: fsmap foofs:1 {0=foofs.smithi028.srtgev=up:creating} 2024-09-16T08:12:28.023 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:12:27 smithi028 ceph-mon[23744]: daemon mds.foofs.smithi028.srtgev is now active in filesystem foofs as rank 0 2024-09-16T08:12:28.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:12:27 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' 2024-09-16T08:12:28.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:12:27 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi144.oozgsi", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-09-16T08:12:28.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:12:27 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi144.oozgsi", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-09-16T08:12:28.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:12:27 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T08:12:28.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:12:27 smithi144 ceph-mon[25668]: Deploying daemon mds.foofs.smithi144.oozgsi on smithi144 2024-09-16T08:12:28.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:12:27 smithi144 ceph-mon[25668]: mds.? [v2:172.21.15.28:6834/800761142,v1:172.21.15.28:6835/800761142] up:boot 2024-09-16T08:12:28.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:12:27 smithi144 ceph-mon[25668]: daemon mds.foofs.smithi028.srtgev assigned to filesystem foofs as rank 0 (now has 1 ranks) 2024-09-16T08:12:28.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:12:27 smithi144 ceph-mon[25668]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-09-16T08:12:28.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:12:27 smithi144 ceph-mon[25668]: Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2024-09-16T08:12:28.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:12:27 smithi144 ceph-mon[25668]: Cluster is now healthy 2024-09-16T08:12:28.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:12:27 smithi144 ceph-mon[25668]: fsmap foofs:0 1 up:standby 2024-09-16T08:12:28.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:12:27 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi028.srtgev"}]: dispatch 2024-09-16T08:12:28.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:12:27 smithi144 ceph-mon[25668]: fsmap foofs:1 {0=foofs.smithi028.srtgev=up:creating} 2024-09-16T08:12:28.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:12:27 smithi144 ceph-mon[25668]: daemon mds.foofs.smithi028.srtgev is now active in filesystem foofs as rank 0 2024-09-16T08:12:28.904 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:12:28 smithi144 ceph-mon[25668]: pgmap v214: 65 pgs: 65 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2024-09-16T08:12:28.904 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:12:28 smithi144 ceph-mon[25668]: mds.? [v2:172.21.15.28:6834/800761142,v1:172.21.15.28:6835/800761142] up:active 2024-09-16T08:12:28.904 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:12:28 smithi144 ceph-mon[25668]: fsmap foofs:1 {0=foofs.smithi028.srtgev=up:active} 2024-09-16T08:12:28.982 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:12:28 smithi028 ceph-mon[23744]: pgmap v214: 65 pgs: 65 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2024-09-16T08:12:28.982 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:12:28 smithi028 ceph-mon[23744]: mds.? [v2:172.21.15.28:6834/800761142,v1:172.21.15.28:6835/800761142] up:active 2024-09-16T08:12:28.982 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:12:28 smithi028 ceph-mon[23744]: fsmap foofs:1 {0=foofs.smithi028.srtgev=up:active} 2024-09-16T08:12:29.377 INFO:teuthology.orchestra.run.smithi028.stdout: 2024-09-16T08:12:29.377 INFO:teuthology.orchestra.run.smithi028.stdout:[{"placement": {"count": 1}, "service_name": "alertmanager", "service_type": "alertmanager", "status": {"created": "2024-09-16T08:06:03.680757Z", "last_refresh": "2024-09-16T08:09:38.042267Z", "ports": [9093, 9094], "running": 1, "size": 1}}, {"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"created": "2024-09-16T08:05:58.113512Z", "last_refresh": "2024-09-16T08:09:38.042334Z", "running": 2, "size": 2}}, {"placement": {"count": 1}, "service_name": "grafana", "service_type": "grafana", "status": {"created": "2024-09-16T08:06:00.803468Z", "last_refresh": "2024-09-16T08:09:38.042399Z", "ports": [3000], "running": 1, "size": 1}}, {"events": ["2024-09-16T08:12:22.262443Z service:mds.foofs [INFO] \"service was created\""], "placement": {"count": 2}, "service_id": "foofs", "service_name": "mds.foofs", "service_type": "mds", "status": {"created": "2024-09-16T08:12:22.255272Z", "running": 0, "size": 2}}, {"placement": {"count": 2}, "service_name": "mgr", "service_type": "mgr", "status": {"created": "2024-09-16T08:05:56.955187Z", "last_refresh": "2024-09-16T08:09:38.042192Z", "running": 2, "size": 2}}, {"placement": {"count": 2, "hosts": ["smithi028:172.21.15.28=smithi028", "smithi144:172.21.15.144=smithi144"]}, "service_name": "mon", "service_type": "mon", "status": {"created": "2024-09-16T08:06:46.314972Z", "last_refresh": "2024-09-16T08:09:38.042045Z", "running": 2, "size": 2}}, {"placement": {"host_pattern": "*"}, "service_name": "node-exporter", "service_type": "node-exporter", "status": {"created": "2024-09-16T08:06:02.516246Z", "last_refresh": "2024-09-16T08:09:38.042463Z", "ports": [9100], "running": 2, "size": 2}}, {"service_name": "osd", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9", "container_image_name": "quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a", "last_refresh": "2024-09-16T08:09:38.042598Z", "running": 8, "size": 8}, "unmanaged": true}, {"placement": {"count": 1}, "service_name": "prometheus", "service_type": "prometheus", "status": {"created": "2024-09-16T08:05:59.457450Z", "last_refresh": "2024-09-16T08:09:38.042527Z", "ports": [9095], "running": 1, "size": 1}}] 2024-09-16T08:12:30.039 INFO:tasks.cephadm:mds.foofs has 0/2 2024-09-16T08:12:30.736 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:12:30 smithi144 ceph-mon[25668]: from='client.14484 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-16T08:12:30.736 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:12:30 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' 2024-09-16T08:12:30.736 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:12:30 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-16T08:12:30.737 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:12:30 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T08:12:30.737 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:12:30 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-16T08:12:30.737 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:12:30 smithi144 ceph-mon[25668]: pgmap v215: 65 pgs: 65 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2024-09-16T08:12:30.737 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:12:30 smithi144 ceph-mon[25668]: mds.? [v2:172.21.15.144:6832/2267142930,v1:172.21.15.144:6833/2267142930] up:boot 2024-09-16T08:12:30.737 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:12:30 smithi144 ceph-mon[25668]: fsmap foofs:1 {0=foofs.smithi028.srtgev=up:active} 1 up:standby 2024-09-16T08:12:30.737 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:12:30 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi144.oozgsi"}]: dispatch 2024-09-16T08:12:30.737 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:12:30 smithi144 ceph-mon[25668]: fsmap foofs:1 {0=foofs.smithi028.srtgev=up:active} 1 up:standby 2024-09-16T08:12:30.946 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:12:30 smithi028 ceph-mon[23744]: from='client.14484 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-16T08:12:30.947 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:12:30 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' 2024-09-16T08:12:30.947 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:12:30 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-16T08:12:30.947 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:12:30 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T08:12:30.947 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:12:30 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-16T08:12:30.947 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:12:30 smithi028 ceph-mon[23744]: pgmap v215: 65 pgs: 65 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2024-09-16T08:12:30.947 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:12:30 smithi028 ceph-mon[23744]: mds.? [v2:172.21.15.144:6832/2267142930,v1:172.21.15.144:6833/2267142930] up:boot 2024-09-16T08:12:30.947 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:12:30 smithi028 ceph-mon[23744]: fsmap foofs:1 {0=foofs.smithi028.srtgev=up:active} 1 up:standby 2024-09-16T08:12:30.947 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:12:30 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi144.oozgsi"}]: dispatch 2024-09-16T08:12:30.947 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:12:30 smithi028 ceph-mon[23744]: fsmap foofs:1 {0=foofs.smithi028.srtgev=up:active} 1 up:standby 2024-09-16T08:12:31.040 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5622b2fe-7402-11ef-bceb-c7b262605968 -- ceph orch ls -f json 2024-09-16T08:12:32.697 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:12:32 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' 2024-09-16T08:12:32.697 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:12:32 smithi028 ceph-mon[23744]: pgmap v216: 65 pgs: 65 active+clean; 578 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 365 B/s wr, 3 op/s 2024-09-16T08:12:32.831 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:12:32 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' 2024-09-16T08:12:32.831 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:12:32 smithi144 ceph-mon[25668]: pgmap v216: 65 pgs: 65 active+clean; 578 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 365 B/s wr, 3 op/s 2024-09-16T08:12:33.372 INFO:teuthology.orchestra.run.smithi028.stdout: 2024-09-16T08:12:33.372 INFO:teuthology.orchestra.run.smithi028.stdout:[{"placement": {"count": 1}, "service_name": "alertmanager", "service_type": "alertmanager", "status": {"created": "2024-09-16T08:06:03.680757Z", "last_refresh": "2024-09-16T08:09:38.042267Z", "ports": [9093, 9094], "running": 1, "size": 1}}, {"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"created": "2024-09-16T08:05:58.113512Z", "last_refresh": "2024-09-16T08:09:38.042334Z", "running": 2, "size": 2}}, {"placement": {"count": 1}, "service_name": "grafana", "service_type": "grafana", "status": {"created": "2024-09-16T08:06:00.803468Z", "last_refresh": "2024-09-16T08:09:38.042399Z", "ports": [3000], "running": 1, "size": 1}}, {"events": ["2024-09-16T08:12:22.262443Z service:mds.foofs [INFO] \"service was created\""], "placement": {"count": 2}, "service_id": "foofs", "service_name": "mds.foofs", "service_type": "mds", "status": {"created": "2024-09-16T08:12:22.255272Z", "last_refresh": "2024-09-16T08:12:31.423941Z", "running": 1, "size": 2}}, {"placement": {"count": 2}, "service_name": "mgr", "service_type": "mgr", "status": {"created": "2024-09-16T08:05:56.955187Z", "last_refresh": "2024-09-16T08:09:38.042192Z", "running": 2, "size": 2}}, {"placement": {"count": 2, "hosts": ["smithi028:172.21.15.28=smithi028", "smithi144:172.21.15.144=smithi144"]}, "service_name": "mon", "service_type": "mon", "status": {"created": "2024-09-16T08:06:46.314972Z", "last_refresh": "2024-09-16T08:09:38.042045Z", "running": 2, "size": 2}}, {"placement": {"host_pattern": "*"}, "service_name": "node-exporter", "service_type": "node-exporter", "status": {"created": "2024-09-16T08:06:02.516246Z", "last_refresh": "2024-09-16T08:09:38.042463Z", "ports": [9100], "running": 2, "size": 2}}, {"service_name": "osd", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9", "container_image_name": "quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a", "last_refresh": "2024-09-16T08:09:38.042598Z", "running": 8, "size": 8}, "unmanaged": true}, {"placement": {"count": 1}, "service_name": "prometheus", "service_type": "prometheus", "status": {"created": "2024-09-16T08:05:59.457450Z", "last_refresh": "2024-09-16T08:09:38.042527Z", "ports": [9095], "running": 1, "size": 1}}] 2024-09-16T08:12:33.970 INFO:tasks.cephadm:mds.foofs has 1/2 2024-09-16T08:12:34.946 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:12:34 smithi028 ceph-mon[23744]: from='client.14492 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-16T08:12:34.947 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:12:34 smithi028 ceph-mon[23744]: pgmap v217: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s wr, 4 op/s 2024-09-16T08:12:34.947 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:12:34 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' 2024-09-16T08:12:34.972 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5622b2fe-7402-11ef-bceb-c7b262605968 -- ceph orch ls -f json 2024-09-16T08:12:35.081 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:12:34 smithi144 ceph-mon[25668]: from='client.14492 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-16T08:12:35.081 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:12:34 smithi144 ceph-mon[25668]: pgmap v217: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s wr, 4 op/s 2024-09-16T08:12:35.081 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:12:34 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' 2024-09-16T08:12:37.303 INFO:teuthology.orchestra.run.smithi028.stdout: 2024-09-16T08:12:37.303 INFO:teuthology.orchestra.run.smithi028.stdout:[{"placement": {"count": 1}, "service_name": "alertmanager", "service_type": "alertmanager", "status": {"created": "2024-09-16T08:06:03.680757Z", "last_refresh": "2024-09-16T08:09:38.042267Z", "ports": [9093, 9094], "running": 1, "size": 1}}, {"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"created": "2024-09-16T08:05:58.113512Z", "last_refresh": "2024-09-16T08:09:38.042334Z", "running": 2, "size": 2}}, {"placement": {"count": 1}, "service_name": "grafana", "service_type": "grafana", "status": {"created": "2024-09-16T08:06:00.803468Z", "last_refresh": "2024-09-16T08:09:38.042399Z", "ports": [3000], "running": 1, "size": 1}}, {"events": ["2024-09-16T08:12:22.262443Z service:mds.foofs [INFO] \"service was created\""], "placement": {"count": 2}, "service_id": "foofs", "service_name": "mds.foofs", "service_type": "mds", "status": {"created": "2024-09-16T08:12:22.255272Z", "last_refresh": "2024-09-16T08:12:31.423941Z", "running": 1, "size": 2}}, {"placement": {"count": 2}, "service_name": "mgr", "service_type": "mgr", "status": {"created": "2024-09-16T08:05:56.955187Z", "last_refresh": "2024-09-16T08:09:38.042192Z", "running": 2, "size": 2}}, {"placement": {"count": 2, "hosts": ["smithi028:172.21.15.28=smithi028", "smithi144:172.21.15.144=smithi144"]}, "service_name": "mon", "service_type": "mon", "status": {"created": "2024-09-16T08:06:46.314972Z", "last_refresh": "2024-09-16T08:09:38.042045Z", "running": 2, "size": 2}}, {"placement": {"host_pattern": "*"}, "service_name": "node-exporter", "service_type": "node-exporter", "status": {"created": "2024-09-16T08:06:02.516246Z", "last_refresh": "2024-09-16T08:09:38.042463Z", "ports": [9100], "running": 2, "size": 2}}, {"service_name": "osd", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9", "container_image_name": "quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a", "last_refresh": "2024-09-16T08:09:38.042598Z", "running": 8, "size": 8}, "unmanaged": true}, {"placement": {"count": 1}, "service_name": "prometheus", "service_type": "prometheus", "status": {"created": "2024-09-16T08:05:59.457450Z", "last_refresh": "2024-09-16T08:09:38.042527Z", "ports": [9095], "running": 1, "size": 1}}] 2024-09-16T08:12:37.303 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:12:37 smithi028 ceph-mon[23744]: pgmap v218: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-09-16T08:12:37.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:12:37 smithi144 ceph-mon[25668]: pgmap v218: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-09-16T08:12:38.062 INFO:tasks.cephadm:mds.foofs has 1/2 2024-09-16T08:12:38.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:12:38 smithi144 ceph-mon[25668]: from='client.14496 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-16T08:12:38.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:12:38 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' 2024-09-16T08:12:38.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:12:38 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' 2024-09-16T08:12:38.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:12:38 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-16T08:12:38.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:12:38 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T08:12:38.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:12:38 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-16T08:12:38.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:12:38 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' 2024-09-16T08:12:38.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:12:38 smithi144 ceph-mon[25668]: pgmap v219: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-09-16T08:12:38.697 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:12:38 smithi028 ceph-mon[23744]: from='client.14496 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-16T08:12:38.697 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:12:38 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' 2024-09-16T08:12:38.697 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:12:38 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' 2024-09-16T08:12:38.697 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:12:38 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-16T08:12:38.697 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:12:38 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T08:12:38.697 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:12:38 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-16T08:12:38.697 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:12:38 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' 2024-09-16T08:12:38.697 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:12:38 smithi028 ceph-mon[23744]: pgmap v219: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-09-16T08:12:39.063 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5622b2fe-7402-11ef-bceb-c7b262605968 -- ceph orch ls -f json 2024-09-16T08:12:40.938 INFO:teuthology.orchestra.run.smithi028.stdout: 2024-09-16T08:12:40.938 INFO:teuthology.orchestra.run.smithi028.stdout:[{"placement": {"count": 1}, "service_name": "alertmanager", "service_type": "alertmanager", "status": {"created": "2024-09-16T08:06:03.680757Z", "last_refresh": "2024-09-16T08:12:37.309979Z", "ports": [9093, 9094], "running": 1, "size": 1}}, {"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"created": "2024-09-16T08:05:58.113512Z", "last_refresh": "2024-09-16T08:12:31.423325Z", "running": 2, "size": 2}}, {"placement": {"count": 1}, "service_name": "grafana", "service_type": "grafana", "status": {"created": "2024-09-16T08:06:00.803468Z", "last_refresh": "2024-09-16T08:12:37.310111Z", "ports": [3000], "running": 1, "size": 1}}, {"events": ["2024-09-16T08:12:22.262443Z service:mds.foofs [INFO] \"service was created\""], "placement": {"count": 2}, "service_id": "foofs", "service_name": "mds.foofs", "service_type": "mds", "status": {"created": "2024-09-16T08:12:22.255272Z", "last_refresh": "2024-09-16T08:12:31.423941Z", "running": 2, "size": 2}}, {"placement": {"count": 2}, "service_name": "mgr", "service_type": "mgr", "status": {"created": "2024-09-16T08:05:56.955187Z", "last_refresh": "2024-09-16T08:12:31.423468Z", "running": 2, "size": 2}}, {"placement": {"count": 2, "hosts": ["smithi028:172.21.15.28=smithi028", "smithi144:172.21.15.144=smithi144"]}, "service_name": "mon", "service_type": "mon", "status": {"created": "2024-09-16T08:06:46.314972Z", "last_refresh": "2024-09-16T08:12:31.423542Z", "running": 2, "size": 2}}, {"placement": {"host_pattern": "*"}, "service_name": "node-exporter", "service_type": "node-exporter", "status": {"created": "2024-09-16T08:06:02.516246Z", "last_refresh": "2024-09-16T08:12:31.423622Z", "ports": [9100], "running": 2, "size": 2}}, {"service_name": "osd", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9", "container_image_name": "quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a", "last_refresh": "2024-09-16T08:12:31.423690Z", "running": 8, "size": 8}, "unmanaged": true}, {"placement": {"count": 1}, "service_name": "prometheus", "service_type": "prometheus", "status": {"created": "2024-09-16T08:05:59.457450Z", "last_refresh": "2024-09-16T08:12:37.310237Z", "ports": [9095], "running": 1, "size": 1}}] 2024-09-16T08:12:41.197 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:12:41 smithi028 ceph-mon[23744]: pgmap v220: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-09-16T08:12:41.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:12:41 smithi144 ceph-mon[25668]: pgmap v220: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-09-16T08:12:41.661 INFO:tasks.cephadm:mds.foofs has 2/2 2024-09-16T08:12:41.661 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-09-16T08:12:41.674 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi028.front.sepia.ceph.com 2024-09-16T08:12:41.674 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5622b2fe-7402-11ef-bceb-c7b262605968 -- bash -c 'ceph nfs cluster create foo --placement=2 || ceph nfs cluster create cephfs foo --placement=2' 2024-09-16T08:12:42.250 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:12:42 smithi028 ceph-mon[23744]: from='client.14500 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-16T08:12:42.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:12:42 smithi144 ceph-mon[25668]: from='client.14500 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-16T08:12:43.313 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:12:43 smithi028 ceph-mon[23744]: pgmap v221: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-09-16T08:12:43.442 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:12:43 smithi144 ceph-mon[25668]: pgmap v221: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-09-16T08:12:44.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:12:44 smithi144 ceph-mon[25668]: pgmap v222: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 853 B/s wr, 1 op/s 2024-09-16T08:12:44.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:12:44 smithi144 ceph-mon[25668]: from='client.14504 -' entity='client.admin' cmd=[{"prefix": "nfs cluster create", "cluster_id": "foo", "placement": "2", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T08:12:44.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:12:44 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "osd pool create", "pool": ".nfs"}]: dispatch 2024-09-16T08:12:44.697 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:12:44 smithi028 ceph-mon[23744]: pgmap v222: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 853 B/s wr, 1 op/s 2024-09-16T08:12:44.698 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:12:44 smithi028 ceph-mon[23744]: from='client.14504 -' entity='client.admin' cmd=[{"prefix": "nfs cluster create", "cluster_id": "foo", "placement": "2", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T08:12:44.698 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:12:44 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "osd pool create", "pool": ".nfs"}]: dispatch 2024-09-16T08:12:45.282 INFO:teuthology.orchestra.run.smithi028.stdout:NFS Cluster Created Successfully 2024-09-16T08:12:45.516 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:12:45 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd='[{"prefix": "osd pool create", "pool": ".nfs"}]': finished 2024-09-16T08:12:45.516 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:12:45 smithi028 ceph-mon[23744]: osdmap e65: 8 total, 8 up, 8 in 2024-09-16T08:12:45.516 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:12:45 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "osd pool application enable", "pool": ".nfs", "app": "nfs"}]: dispatch 2024-09-16T08:12:45.516 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:12:45 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd='[{"prefix": "osd pool application enable", "pool": ".nfs", "app": "nfs"}]': finished 2024-09-16T08:12:45.516 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:12:45 smithi028 ceph-mon[23744]: osdmap e66: 8 total, 8 up, 8 in 2024-09-16T08:12:45.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:12:45 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd='[{"prefix": "osd pool create", "pool": ".nfs"}]': finished 2024-09-16T08:12:45.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:12:45 smithi144 ceph-mon[25668]: osdmap e65: 8 total, 8 up, 8 in 2024-09-16T08:12:45.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:12:45 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "osd pool application enable", "pool": ".nfs", "app": "nfs"}]: dispatch 2024-09-16T08:12:45.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:12:45 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd='[{"prefix": "osd pool application enable", "pool": ".nfs", "app": "nfs"}]': finished 2024-09-16T08:12:45.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:12:45 smithi144 ceph-mon[25668]: osdmap e66: 8 total, 8 up, 8 in 2024-09-16T08:12:46.012 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5622b2fe-7402-11ef-bceb-c7b262605968 -- bash -c 'ceph nfs export create cephfs --fsname foofs --clusterid foo --binding /fake || ceph nfs export create cephfs --fsname foofs --cluster-id foo --pseudo-path /fake' 2024-09-16T08:12:46.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:12:46 smithi144 ceph-mon[25668]: Saving service nfs.foo spec with placement count:2 2024-09-16T08:12:46.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:12:46 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' 2024-09-16T08:12:46.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:12:46 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-16T08:12:46.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:12:46 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T08:12:46.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:12:46 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-16T08:12:46.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:12:46 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' 2024-09-16T08:12:46.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:12:46 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' 2024-09-16T08:12:46.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:12:46 smithi144 ceph-mon[25668]: Creating key for client.nfs.foo.0.0.smithi028.dkttgj 2024-09-16T08:12:46.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:12:46 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi028.dkttgj", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]: dispatch 2024-09-16T08:12:46.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:12:46 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi028.dkttgj", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]': finished 2024-09-16T08:12:46.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:12:46 smithi144 ceph-mon[25668]: Ensuring nfs.foo.0 is in the ganesha grace table 2024-09-16T08:12:46.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:12:46 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]: dispatch 2024-09-16T08:12:46.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:12:46 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-09-16T08:12:46.583 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:12:46 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T08:12:46.583 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:12:46 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]: dispatch 2024-09-16T08:12:46.583 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:12:46 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-09-16T08:12:46.583 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:12:46 smithi144 ceph-mon[25668]: Rados config object exists: conf-nfs.foo 2024-09-16T08:12:46.583 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:12:46 smithi144 ceph-mon[25668]: Creating key for client.nfs.foo.0.0.smithi028.dkttgj-rgw 2024-09-16T08:12:46.583 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:12:46 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi028.dkttgj-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-09-16T08:12:46.583 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:12:46 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi028.dkttgj-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-09-16T08:12:46.583 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:12:46 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T08:12:46.583 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:12:46 smithi144 ceph-mon[25668]: Deploying daemon nfs.foo.0.0.smithi028.dkttgj on smithi028 2024-09-16T08:12:46.583 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:12:46 smithi144 ceph-mon[25668]: pgmap v225: 97 pgs: 4 creating+peering, 28 unknown, 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2024-09-16T08:12:46.583 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:12:46 smithi144 ceph-mon[25668]: osdmap e67: 8 total, 8 up, 8 in 2024-09-16T08:12:46.584 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:12:46 smithi028 ceph-mon[23744]: Saving service nfs.foo spec with placement count:2 2024-09-16T08:12:46.584 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:12:46 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' 2024-09-16T08:12:46.584 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:12:46 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-16T08:12:46.584 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:12:46 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T08:12:46.584 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:12:46 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-16T08:12:46.584 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:12:46 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' 2024-09-16T08:12:46.584 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:12:46 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' 2024-09-16T08:12:46.584 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:12:46 smithi028 ceph-mon[23744]: Creating key for client.nfs.foo.0.0.smithi028.dkttgj 2024-09-16T08:12:46.584 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:12:46 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi028.dkttgj", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]: dispatch 2024-09-16T08:12:46.584 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:12:46 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi028.dkttgj", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]': finished 2024-09-16T08:12:46.584 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:12:46 smithi028 ceph-mon[23744]: Ensuring nfs.foo.0 is in the ganesha grace table 2024-09-16T08:12:46.584 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:12:46 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]: dispatch 2024-09-16T08:12:46.585 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:12:46 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-09-16T08:12:46.585 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:12:46 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T08:12:46.585 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:12:46 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]: dispatch 2024-09-16T08:12:46.585 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:12:46 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-09-16T08:12:46.585 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:12:46 smithi028 ceph-mon[23744]: Rados config object exists: conf-nfs.foo 2024-09-16T08:12:46.585 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:12:46 smithi028 ceph-mon[23744]: Creating key for client.nfs.foo.0.0.smithi028.dkttgj-rgw 2024-09-16T08:12:46.585 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:12:46 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi028.dkttgj-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-09-16T08:12:46.585 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:12:46 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi028.dkttgj-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-09-16T08:12:46.585 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:12:46 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T08:12:46.585 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:12:46 smithi028 ceph-mon[23744]: Deploying daemon nfs.foo.0.0.smithi028.dkttgj on smithi028 2024-09-16T08:12:46.586 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:12:46 smithi028 ceph-mon[23744]: pgmap v225: 97 pgs: 4 creating+peering, 28 unknown, 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2024-09-16T08:12:46.586 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:12:46 smithi028 ceph-mon[23744]: osdmap e67: 8 total, 8 up, 8 in 2024-09-16T08:12:47.765 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:12:47 smithi028 ceph-mon[23744]: mgrmap e20: smithi028.azvyzy(active, since 5m), standbys: smithi144.utzszj 2024-09-16T08:12:47.831 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:12:47 smithi144 ceph-mon[25668]: mgrmap e20: smithi028.azvyzy(active, since 5m), standbys: smithi144.utzszj 2024-09-16T08:12:48.693 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:12:48 smithi028 ceph-mon[23744]: pgmap v227: 97 pgs: 4 creating+peering, 93 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 341 B/s wr, 0 op/s 2024-09-16T08:12:48.831 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:12:48 smithi144 ceph-mon[25668]: pgmap v227: 97 pgs: 4 creating+peering, 93 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 341 B/s wr, 0 op/s 2024-09-16T08:12:49.151 INFO:teuthology.orchestra.run.smithi028.stderr:Invalid command: Unexpected argument '--clusterid' 2024-09-16T08:12:49.152 INFO:teuthology.orchestra.run.smithi028.stderr:nfs export create cephfs [] [--readonly] [--client_addr ...] [--squash ] : Create a CephFS export 2024-09-16T08:12:49.152 INFO:teuthology.orchestra.run.smithi028.stderr:Error EINVAL: invalid command 2024-09-16T08:12:49.526 INFO:teuthology.orchestra.run.smithi028.stdout:{ 2024-09-16T08:12:49.527 INFO:teuthology.orchestra.run.smithi028.stdout: "bind": "/fake", 2024-09-16T08:12:49.527 INFO:teuthology.orchestra.run.smithi028.stdout: "fs": "foofs", 2024-09-16T08:12:49.527 INFO:teuthology.orchestra.run.smithi028.stdout: "path": "/", 2024-09-16T08:12:49.527 INFO:teuthology.orchestra.run.smithi028.stdout: "cluster": "foo", 2024-09-16T08:12:49.527 INFO:teuthology.orchestra.run.smithi028.stdout: "mode": "RW" 2024-09-16T08:12:49.527 INFO:teuthology.orchestra.run.smithi028.stdout:} 2024-09-16T08:12:50.195 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5622b2fe-7402-11ef-bceb-c7b262605968 -- bash -c 'while ! ceph orch ls | grep nfs | grep 2/2 ; do sleep 1 ; done' 2024-09-16T08:12:50.697 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:12:50 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' 2024-09-16T08:12:50.697 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:12:50 smithi028 ceph-mon[23744]: Creating key for client.nfs.foo.1.0.smithi144.epipkg 2024-09-16T08:12:50.697 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:12:50 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi144.epipkg", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]: dispatch 2024-09-16T08:12:50.697 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:12:50 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi144.epipkg", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]': finished 2024-09-16T08:12:50.697 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:12:50 smithi028 ceph-mon[23744]: Ensuring nfs.foo.1 is in the ganesha grace table 2024-09-16T08:12:50.697 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:12:50 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]: dispatch 2024-09-16T08:12:50.697 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:12:50 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-09-16T08:12:50.697 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:12:50 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T08:12:50.697 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:12:50 smithi028 ceph-mon[23744]: from='client.14526 -' entity='client.admin' cmd=[{"prefix": "nfs export create cephfs", "fsname": "foofs", "cluster_id": "foo", "pseudo_path": "/fake", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T08:12:50.697 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:12:50 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo, allow rw tag cephfs data=foofs", "mds", "allow rw path=/"], "format": "json"}]: dispatch 2024-09-16T08:12:50.697 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:12:50 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo, allow rw tag cephfs data=foofs", "mds", "allow rw path=/"], "format": "json"}]': finished 2024-09-16T08:12:50.698 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:12:50 smithi028 ceph-mon[23744]: pgmap v228: 97 pgs: 4 creating+peering, 93 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 341 B/s wr, 0 op/s 2024-09-16T08:12:50.831 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:12:50 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' 2024-09-16T08:12:50.831 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:12:50 smithi144 ceph-mon[25668]: Creating key for client.nfs.foo.1.0.smithi144.epipkg 2024-09-16T08:12:50.831 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:12:50 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi144.epipkg", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]: dispatch 2024-09-16T08:12:50.831 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:12:50 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi144.epipkg", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]': finished 2024-09-16T08:12:50.831 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:12:50 smithi144 ceph-mon[25668]: Ensuring nfs.foo.1 is in the ganesha grace table 2024-09-16T08:12:50.832 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:12:50 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]: dispatch 2024-09-16T08:12:50.832 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:12:50 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-09-16T08:12:50.832 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:12:50 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T08:12:50.832 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:12:50 smithi144 ceph-mon[25668]: from='client.14526 -' entity='client.admin' cmd=[{"prefix": "nfs export create cephfs", "fsname": "foofs", "cluster_id": "foo", "pseudo_path": "/fake", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T08:12:50.832 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:12:50 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo, allow rw tag cephfs data=foofs", "mds", "allow rw path=/"], "format": "json"}]: dispatch 2024-09-16T08:12:50.832 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:12:50 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo, allow rw tag cephfs data=foofs", "mds", "allow rw path=/"], "format": "json"}]': finished 2024-09-16T08:12:50.832 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:12:50 smithi144 ceph-mon[25668]: pgmap v228: 97 pgs: 4 creating+peering, 93 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 341 B/s wr, 0 op/s 2024-09-16T08:12:52.980 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:12:52 smithi144 ceph-mon[25668]: pgmap v229: 97 pgs: 97 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s rd, 1.5 KiB/s wr, 3 op/s 2024-09-16T08:12:52.981 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:12:52 smithi144 ceph-mon[25668]: from='client.14542 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T08:12:52.981 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:12:52 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]: dispatch 2024-09-16T08:12:52.981 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:12:52 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-09-16T08:12:52.981 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:12:52 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi144.epipkg-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-09-16T08:12:52.981 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:12:52 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi144.epipkg-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-09-16T08:12:52.981 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:12:52 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T08:12:53.197 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:12:52 smithi028 ceph-mon[23744]: pgmap v229: 97 pgs: 97 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s rd, 1.5 KiB/s wr, 3 op/s 2024-09-16T08:12:53.197 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:12:52 smithi028 ceph-mon[23744]: from='client.14542 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T08:12:53.197 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:12:52 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]: dispatch 2024-09-16T08:12:53.197 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:12:52 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-09-16T08:12:53.197 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:12:52 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi144.epipkg-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-09-16T08:12:53.197 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:12:52 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi144.epipkg-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-09-16T08:12:53.197 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:12:52 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T08:12:54.081 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:12:53 smithi144 ceph-mon[25668]: Rados config object exists: conf-nfs.foo 2024-09-16T08:12:54.081 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:12:53 smithi144 ceph-mon[25668]: Creating key for client.nfs.foo.1.0.smithi144.epipkg-rgw 2024-09-16T08:12:54.082 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:12:53 smithi144 ceph-mon[25668]: Deploying daemon nfs.foo.1.0.smithi144.epipkg on smithi144 2024-09-16T08:12:54.196 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:12:53 smithi028 ceph-mon[23744]: Rados config object exists: conf-nfs.foo 2024-09-16T08:12:54.197 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:12:53 smithi028 ceph-mon[23744]: Creating key for client.nfs.foo.1.0.smithi144.epipkg-rgw 2024-09-16T08:12:54.197 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:12:53 smithi028 ceph-mon[23744]: Deploying daemon nfs.foo.1.0.smithi144.epipkg on smithi144 2024-09-16T08:12:54.831 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:12:54 smithi144 ceph-mon[25668]: from='client.14556 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T08:12:54.831 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:12:54 smithi144 ceph-mon[25668]: pgmap v230: 97 pgs: 97 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1.3 KiB/s wr, 2 op/s 2024-09-16T08:12:54.831 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:12:54 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' 2024-09-16T08:12:54.831 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:12:54 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-16T08:12:54.831 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:12:54 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T08:12:54.832 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:12:54 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-16T08:12:54.946 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:12:54 smithi028 ceph-mon[23744]: from='client.14556 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T08:12:54.947 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:12:54 smithi028 ceph-mon[23744]: pgmap v230: 97 pgs: 97 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1.3 KiB/s wr, 2 op/s 2024-09-16T08:12:54.947 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:12:54 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' 2024-09-16T08:12:54.947 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:12:54 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-16T08:12:54.947 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:12:54 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T08:12:54.947 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:12:54 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-16T08:12:56.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:12:56 smithi144 ceph-mon[25668]: from='client.14560 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T08:12:56.446 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:12:56 smithi028 ceph-mon[23744]: from='client.14560 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T08:12:57.232 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:12:57 smithi144 ceph-mon[25668]: pgmap v231: 97 pgs: 97 active+clean; 581 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1.9 KiB/s wr, 3 op/s 2024-09-16T08:12:57.232 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:12:57 smithi144 ceph-mon[25668]: from='client.14576 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T08:12:57.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:12:57 smithi028 ceph-mon[23744]: pgmap v231: 97 pgs: 97 active+clean; 581 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1.9 KiB/s wr, 3 op/s 2024-09-16T08:12:57.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:12:57 smithi028 ceph-mon[23744]: from='client.14576 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T08:12:58.697 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:12:58 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' 2024-09-16T08:12:58.697 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:12:58 smithi028 ceph-mon[23744]: from='client.14580 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T08:12:58.697 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:12:58 smithi028 ceph-mon[23744]: pgmap v232: 97 pgs: 97 active+clean; 581 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 3.6 KiB/s rd, 2.2 KiB/s wr, 6 op/s 2024-09-16T08:12:58.831 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:12:58 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' 2024-09-16T08:12:58.831 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:12:58 smithi144 ceph-mon[25668]: from='client.14580 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T08:12:58.831 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:12:58 smithi144 ceph-mon[25668]: pgmap v232: 97 pgs: 97 active+clean; 581 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 3.6 KiB/s rd, 2.2 KiB/s wr, 6 op/s 2024-09-16T08:12:59.947 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:12:59 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' 2024-09-16T08:12:59.947 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:12:59 smithi028 ceph-mon[23744]: from='client.14584 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T08:12:59.947 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:12:59 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' 2024-09-16T08:12:59.947 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:12:59 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' 2024-09-16T08:12:59.947 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:12:59 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' 2024-09-16T08:12:59.947 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:12:59 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-16T08:12:59.947 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:12:59 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T08:12:59.947 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:12:59 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-16T08:12:59.947 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:12:59 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' 2024-09-16T08:12:59.947 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:12:59 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' 2024-09-16T08:13:00.048 INFO:teuthology.orchestra.run.smithi028.stdout:nfs.foo ?:2049 2/2 2s ago 14s count:2 2024-09-16T08:13:00.081 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:12:59 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' 2024-09-16T08:13:00.081 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:12:59 smithi144 ceph-mon[25668]: from='client.14584 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T08:13:00.081 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:12:59 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' 2024-09-16T08:13:00.082 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:12:59 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' 2024-09-16T08:13:00.082 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:12:59 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' 2024-09-16T08:13:00.082 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:12:59 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-16T08:13:00.082 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:12:59 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T08:13:00.082 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:12:59 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-16T08:13:00.082 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:12:59 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' 2024-09-16T08:13:00.082 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:12:59 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' 2024-09-16T08:13:00.750 INFO:teuthology.run_tasks:Running task vip.exec... 2024-09-16T08:13:00.763 INFO:tasks.vip:Running commands on role host.a host ubuntu@smithi028.front.sepia.ceph.com 2024-09-16T08:13:00.763 DEBUG:teuthology.orchestra.run.smithi028:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c 'mkdir /mnt/foo' 2024-09-16T08:13:00.791 INFO:teuthology.orchestra.run.smithi028.stderr:+ mkdir /mnt/foo 2024-09-16T08:13:00.806 DEBUG:teuthology.orchestra.run.smithi028:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c 'while ! mount -t nfs $(hostname):/fake /mnt/foo -o sync ; do sleep 5 ; done' 2024-09-16T08:13:00.872 INFO:teuthology.orchestra.run.smithi028.stderr:++ hostname 2024-09-16T08:13:00.873 INFO:teuthology.orchestra.run.smithi028.stderr:+ mount -t nfs smithi028:/fake /mnt/foo -o sync 2024-09-16T08:13:01.343 DEBUG:teuthology.orchestra.run.smithi028:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c 'echo test > /mnt/foo/testfile' 2024-09-16T08:13:01.407 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:13:01 smithi028 ceph-mon[23744]: pgmap v233: 97 pgs: 97 active+clean; 581 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 3.4 KiB/s rd, 2.1 KiB/s wr, 5 op/s 2024-09-16T08:13:01.407 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:13:01 smithi028 ceph-mon[23744]: from='client.14588 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T08:13:01.409 INFO:teuthology.orchestra.run.smithi028.stderr:+ echo test 2024-09-16T08:13:01.420 DEBUG:teuthology.orchestra.run.smithi028:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c sync 2024-09-16T08:13:01.485 INFO:teuthology.orchestra.run.smithi028.stderr:+ sync 2024-09-16T08:13:01.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:13:01 smithi144 ceph-mon[25668]: pgmap v233: 97 pgs: 97 active+clean; 581 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 3.4 KiB/s rd, 2.1 KiB/s wr, 5 op/s 2024-09-16T08:13:01.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:13:01 smithi144 ceph-mon[25668]: from='client.14588 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T08:13:01.804 INFO:teuthology.run_tasks:Running task parallel... 2024-09-16T08:13:01.817 INFO:teuthology.task.parallel:starting parallel... 2024-09-16T08:13:01.818 INFO:teuthology.task.parallel:In parallel, running task sequential... 2024-09-16T08:13:01.819 INFO:teuthology.task.sequential:In sequential, running task cephadm.shell... 2024-09-16T08:13:01.820 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi028.front.sepia.ceph.com 2024-09-16T08:13:01.820 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5622b2fe-7402-11ef-bceb-c7b262605968 -e sha1=8293d73f8690540e843a81caec373f9cc29cf705 -- bash -c 'ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force' 2024-09-16T08:13:01.821 INFO:teuthology.task.parallel:In parallel, running task sequential... 2024-09-16T08:13:01.821 INFO:teuthology.task.sequential:In sequential, running task exec... 2024-09-16T08:13:01.822 INFO:teuthology.task.exec:Executing custom commands... 2024-09-16T08:13:01.822 INFO:teuthology.task.exec:Running commands on role host.a host ubuntu@smithi028.front.sepia.ceph.com 2024-09-16T08:13:01.822 DEBUG:teuthology.orchestra.run.smithi028:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'cd /mnt/foo && dbench 5 -t 600 || true' 2024-09-16T08:13:01.882 INFO:teuthology.orchestra.run.smithi028.stdout:dbench version 4.00 - Copyright Andrew Tridgell 1999-2004 2024-09-16T08:13:01.883 INFO:teuthology.orchestra.run.smithi028.stdout: 2024-09-16T08:13:01.883 INFO:teuthology.orchestra.run.smithi028.stdout:Running for 600 seconds with load '/usr/share/dbench/client.txt' and minimum warmup 120 secs 2024-09-16T08:13:01.884 INFO:teuthology.orchestra.run.smithi028.stdout:4 of 5 processes prepared for launch 0 sec 2024-09-16T08:13:01.984 INFO:teuthology.orchestra.run.smithi028.stdout:5 of 5 processes prepared for launch 0 sec 2024-09-16T08:13:01.984 INFO:teuthology.orchestra.run.smithi028.stdout:releasing clients 2024-09-16T08:13:02.984 INFO:teuthology.orchestra.run.smithi028.stdout: 5 134 29.47 MB/sec warmup 1 sec latency 45.508 ms 2024-09-16T08:13:03.222 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:13:03 smithi028 ceph-mon[23744]: pgmap v234: 97 pgs: 97 active+clean; 581 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 3.4 KiB/s rd, 2.1 KiB/s wr, 5 op/s 2024-09-16T08:13:03.442 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:13:03 smithi144 ceph-mon[25668]: pgmap v234: 97 pgs: 97 active+clean; 581 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 3.4 KiB/s rd, 2.1 KiB/s wr, 5 op/s 2024-09-16T08:13:03.985 INFO:teuthology.orchestra.run.smithi028.stdout: 5 270 25.36 MB/sec warmup 2 sec latency 32.481 ms 2024-09-16T08:13:04.238 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:13:04 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.1b", "id": [7, 2]}]: dispatch 2024-09-16T08:13:04.238 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:13:04 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]: dispatch 2024-09-16T08:13:04.238 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:13:04 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.f", "id": [1, 2]}]: dispatch 2024-09-16T08:13:04.238 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:13:04 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [1, 2]}]: dispatch 2024-09-16T08:13:04.238 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:13:04 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi028.azvyzy/trash_purge_schedule"}]: dispatch 2024-09-16T08:13:04.238 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:13:04 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi028.azvyzy/mirror_snapshot_schedule"}]: dispatch 2024-09-16T08:13:04.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:13:04 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.1b", "id": [7, 2]}]: dispatch 2024-09-16T08:13:04.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:13:04 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]: dispatch 2024-09-16T08:13:04.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:13:04 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.f", "id": [1, 2]}]: dispatch 2024-09-16T08:13:04.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:13:04 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [1, 2]}]: dispatch 2024-09-16T08:13:04.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:13:04 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi028.azvyzy/trash_purge_schedule"}]: dispatch 2024-09-16T08:13:04.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:13:04 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi028.azvyzy/mirror_snapshot_schedule"}]: dispatch 2024-09-16T08:13:04.985 INFO:teuthology.orchestra.run.smithi028.stdout: 5 402 23.67 MB/sec warmup 3 sec latency 209.299 ms 2024-09-16T08:13:05.030 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5622b2fe-7402-11ef-bceb-c7b262605968 -e sha1=8293d73f8690540e843a81caec373f9cc29cf705 -- bash -c 'ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force' 2024-09-16T08:13:05.264 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:13:05 smithi028 ceph-mon[23744]: pgmap v235: 97 pgs: 97 active+clean; 2.1 MiB data, 51 MiB used, 715 GiB / 715 GiB avail; 2.6 KiB/s rd, 130 KiB/s wr, 6 op/s 2024-09-16T08:13:05.265 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:13:05 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.1b", "id": [7, 2]}]': finished 2024-09-16T08:13:05.265 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:13:05 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]': finished 2024-09-16T08:13:05.265 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:13:05 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.f", "id": [1, 2]}]': finished 2024-09-16T08:13:05.265 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:13:05 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [1, 2]}]': finished 2024-09-16T08:13:05.265 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:13:05 smithi028 ceph-mon[23744]: osdmap e68: 8 total, 8 up, 8 in 2024-09-16T08:13:05.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:13:05 smithi144 ceph-mon[25668]: pgmap v235: 97 pgs: 97 active+clean; 2.1 MiB data, 51 MiB used, 715 GiB / 715 GiB avail; 2.6 KiB/s rd, 130 KiB/s wr, 6 op/s 2024-09-16T08:13:05.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:13:05 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.1b", "id": [7, 2]}]': finished 2024-09-16T08:13:05.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:13:05 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]': finished 2024-09-16T08:13:05.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:13:05 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.f", "id": [1, 2]}]': finished 2024-09-16T08:13:05.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:13:05 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [1, 2]}]': finished 2024-09-16T08:13:05.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:13:05 smithi144 ceph-mon[25668]: osdmap e68: 8 total, 8 up, 8 in 2024-09-16T08:13:05.984 INFO:teuthology.orchestra.run.smithi028.stdout: 5 633 26.29 MB/sec warmup 4 sec latency 371.702 ms 2024-09-16T08:13:06.277 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:13:06 smithi028 ceph-mon[23744]: osdmap e69: 8 total, 8 up, 8 in 2024-09-16T08:13:06.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:13:06 smithi144 ceph-mon[25668]: osdmap e69: 8 total, 8 up, 8 in 2024-09-16T08:13:06.985 INFO:teuthology.orchestra.run.smithi028.stdout: 5 1333 24.28 MB/sec warmup 5 sec latency 54.292 ms 2024-09-16T08:13:07.282 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:13:07 smithi028 ceph-mon[23744]: pgmap v238: 97 pgs: 97 active+clean; 24 MiB data, 66 MiB used, 715 GiB / 715 GiB avail; 3.1 MiB/s wr, 242 op/s 2024-09-16T08:13:07.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:13:07 smithi144 ceph-mon[25668]: pgmap v238: 97 pgs: 97 active+clean; 24 MiB data, 66 MiB used, 715 GiB / 715 GiB avail; 3.1 MiB/s wr, 242 op/s 2024-09-16T08:13:07.985 INFO:teuthology.orchestra.run.smithi028.stdout: 5 2440 23.55 MB/sec warmup 6 sec latency 10.282 ms 2024-09-16T08:13:08.985 INFO:teuthology.orchestra.run.smithi028.stdout: 5 3459 26.20 MB/sec warmup 7 sec latency 11.154 ms 2024-09-16T08:13:09.293 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:13:09 smithi028 ceph-mon[23744]: pgmap v239: 97 pgs: 97 active+clean; 101 MiB data, 125 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 13 MiB/s wr, 561 op/s 2024-09-16T08:13:09.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:13:09 smithi144 ceph-mon[25668]: pgmap v239: 97 pgs: 97 active+clean; 101 MiB data, 125 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 13 MiB/s wr, 561 op/s 2024-09-16T08:13:09.985 INFO:teuthology.orchestra.run.smithi028.stdout: 5 4150 25.10 MB/sec warmup 8 sec latency 17.139 ms 2024-09-16T08:13:09.993 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5622b2fe-7402-11ef-bceb-c7b262605968 -e sha1=8293d73f8690540e843a81caec373f9cc29cf705 -- bash -c 'ceph config set global log_to_journald false --force' 2024-09-16T08:13:10.985 INFO:teuthology.orchestra.run.smithi028.stdout: 5 5068 25.11 MB/sec warmup 9 sec latency 25.068 ms 2024-09-16T08:13:11.307 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:13:11 smithi028 ceph-mon[23744]: pgmap v240: 97 pgs: 97 active+clean; 101 MiB data, 125 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 13 MiB/s wr, 561 op/s 2024-09-16T08:13:11.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:13:11 smithi144 ceph-mon[25668]: pgmap v240: 97 pgs: 97 active+clean; 101 MiB data, 125 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 13 MiB/s wr, 561 op/s 2024-09-16T08:13:11.989 INFO:teuthology.orchestra.run.smithi028.stdout: 5 5909 23.44 MB/sec warmup 10 sec latency 31.516 ms 2024-09-16T08:13:12.985 INFO:teuthology.orchestra.run.smithi028.stdout: 5 6368 23.17 MB/sec warmup 11 sec latency 45.138 ms 2024-09-16T08:13:13.339 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:13:13 smithi028 ceph-mon[23744]: pgmap v241: 97 pgs: 97 active+clean; 122 MiB data, 192 MiB used, 715 GiB / 715 GiB avail; 2.0 KiB/s rd, 17 MiB/s wr, 924 op/s 2024-09-16T08:13:13.441 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:13:13 smithi144 ceph-mon[25668]: pgmap v241: 97 pgs: 97 active+clean; 122 MiB data, 192 MiB used, 715 GiB / 715 GiB avail; 2.0 KiB/s rd, 17 MiB/s wr, 924 op/s 2024-09-16T08:13:13.985 INFO:teuthology.orchestra.run.smithi028.stdout: 5 6971 23.21 MB/sec warmup 12 sec latency 34.388 ms 2024-09-16T08:13:14.609 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5622b2fe-7402-11ef-bceb-c7b262605968 -e sha1=8293d73f8690540e843a81caec373f9cc29cf705 -- bash -c 'ceph mgr module enable nfs --force' 2024-09-16T08:13:14.987 INFO:teuthology.orchestra.run.smithi028.stdout: 5 7365 22.65 MB/sec warmup 13 sec latency 63.826 ms 2024-09-16T08:13:15.371 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:13:15 smithi028 ceph-mon[23744]: pgmap v242: 97 pgs: 97 active+clean; 135 MiB data, 248 MiB used, 715 GiB / 715 GiB avail; 1.7 KiB/s rd, 18 MiB/s wr, 987 op/s 2024-09-16T08:13:15.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:13:15 smithi144 ceph-mon[25668]: pgmap v242: 97 pgs: 97 active+clean; 135 MiB data, 248 MiB used, 715 GiB / 715 GiB avail; 1.7 KiB/s rd, 18 MiB/s wr, 987 op/s 2024-09-16T08:13:15.985 INFO:teuthology.orchestra.run.smithi028.stdout: 5 7737 21.20 MB/sec warmup 14 sec latency 44.305 ms 2024-09-16T08:13:16.985 INFO:teuthology.orchestra.run.smithi028.stdout: 5 8025 19.99 MB/sec warmup 15 sec latency 68.718 ms 2024-09-16T08:13:17.394 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:13:17 smithi028 ceph-mon[23744]: pgmap v243: 97 pgs: 97 active+clean; 141 MiB data, 280 MiB used, 715 GiB / 715 GiB avail; 8.6 KiB/s rd, 16 MiB/s wr, 910 op/s 2024-09-16T08:13:17.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:13:17 smithi144 ceph-mon[25668]: pgmap v243: 97 pgs: 97 active+clean; 141 MiB data, 280 MiB used, 715 GiB / 715 GiB avail; 8.6 KiB/s rd, 16 MiB/s wr, 910 op/s 2024-09-16T08:13:17.992 INFO:teuthology.orchestra.run.smithi028.stdout: 5 8782 20.13 MB/sec warmup 16 sec latency 23.378 ms 2024-09-16T08:13:18.237 INFO:teuthology.orchestra.run.smithi028.stderr:module 'nfs' is already enabled 2024-09-16T08:13:18.399 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:13:18 smithi028 ceph-mon[23744]: from='client.? 172.21.15.28:0/2079821298' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "nfs", "force": true}]: dispatch 2024-09-16T08:13:18.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:13:18 smithi144 ceph-mon[25668]: from='client.? 172.21.15.28:0/2079821298' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "nfs", "force": true}]: dispatch 2024-09-16T08:13:18.992 INFO:teuthology.orchestra.run.smithi028.stdout: 5 9433 19.39 MB/sec warmup 17 sec latency 55.130 ms 2024-09-16T08:13:19.227 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:13:19 smithi028 ceph-mon[23744]: pgmap v244: 97 pgs: 97 active+clean; 142 MiB data, 337 MiB used, 715 GiB / 715 GiB avail; 7.7 KiB/s rd, 15 MiB/s wr, 948 op/s 2024-09-16T08:13:19.228 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:13:19 smithi028 ceph-mon[23744]: from='client.? 172.21.15.28:0/2079821298' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "nfs", "force": true}]': finished 2024-09-16T08:13:19.228 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:13:19 smithi028 ceph-mon[23744]: mgrmap e21: smithi028.azvyzy(active, since 6m), standbys: smithi144.utzszj 2024-09-16T08:13:19.338 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5622b2fe-7402-11ef-bceb-c7b262605968 -e sha1=8293d73f8690540e843a81caec373f9cc29cf705 -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1' 2024-09-16T08:13:19.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:13:19 smithi144 ceph-mon[25668]: pgmap v244: 97 pgs: 97 active+clean; 142 MiB data, 337 MiB used, 715 GiB / 715 GiB avail; 7.7 KiB/s rd, 15 MiB/s wr, 948 op/s 2024-09-16T08:13:19.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:13:19 smithi144 ceph-mon[25668]: from='client.? 172.21.15.28:0/2079821298' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "nfs", "force": true}]': finished 2024-09-16T08:13:19.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:13:19 smithi144 ceph-mon[25668]: mgrmap e21: smithi028.azvyzy(active, since 6m), standbys: smithi144.utzszj 2024-09-16T08:13:19.992 INFO:teuthology.orchestra.run.smithi028.stdout: 5 9658 18.64 MB/sec warmup 18 sec latency 38.224 ms 2024-09-16T08:13:20.992 INFO:teuthology.orchestra.run.smithi028.stdout: 5 10072 18.84 MB/sec warmup 19 sec latency 50.605 ms 2024-09-16T08:13:21.275 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:13:21 smithi028 ceph-mon[23744]: pgmap v245: 97 pgs: 97 active+clean; 142 MiB data, 337 MiB used, 715 GiB / 715 GiB avail; 7.6 KiB/s rd, 8.6 MiB/s wr, 735 op/s 2024-09-16T08:13:21.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:13:21 smithi144 ceph-mon[25668]: pgmap v245: 97 pgs: 97 active+clean; 142 MiB data, 337 MiB used, 715 GiB / 715 GiB avail; 7.6 KiB/s rd, 8.6 MiB/s wr, 735 op/s 2024-09-16T08:13:21.985 INFO:teuthology.orchestra.run.smithi028.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:8293d73f8690540e843a81caec373f9cc29cf705 2024-09-16T08:13:21.992 INFO:teuthology.orchestra.run.smithi028.stdout: 5 10622 19.01 MB/sec warmup 20 sec latency 34.864 ms 2024-09-16T08:13:22.992 INFO:teuthology.orchestra.run.smithi028.stdout: 5 10984 18.60 MB/sec warmup 21 sec latency 40.212 ms 2024-09-16T08:13:23.049 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:13:22 smithi028 ceph-mon[23744]: pgmap v246: 97 pgs: 97 active+clean; 146 MiB data, 369 MiB used, 715 GiB / 715 GiB avail; 12 KiB/s rd, 10 MiB/s wr, 956 op/s 2024-09-16T08:13:23.049 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:13:22 smithi028 ceph-mon[23744]: from='client.14608 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:8293d73f8690540e843a81caec373f9cc29cf705", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T08:13:23.049 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:13:22 smithi028 ceph-mon[23744]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:8293d73f8690540e843a81caec373f9cc29cf705 2024-09-16T08:13:23.049 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:13:22 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' 2024-09-16T08:13:23.049 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:13:22 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-16T08:13:23.049 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:13:22 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T08:13:23.049 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:13:22 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-16T08:13:23.049 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:13:22 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' 2024-09-16T08:13:23.049 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:13:22 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' 2024-09-16T08:13:23.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:13:22 smithi144 ceph-mon[25668]: pgmap v246: 97 pgs: 97 active+clean; 146 MiB data, 369 MiB used, 715 GiB / 715 GiB avail; 12 KiB/s rd, 10 MiB/s wr, 956 op/s 2024-09-16T08:13:23.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:13:22 smithi144 ceph-mon[25668]: from='client.14608 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:8293d73f8690540e843a81caec373f9cc29cf705", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T08:13:23.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:13:22 smithi144 ceph-mon[25668]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:8293d73f8690540e843a81caec373f9cc29cf705 2024-09-16T08:13:23.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:13:22 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' 2024-09-16T08:13:23.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:13:22 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-16T08:13:23.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:13:22 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T08:13:23.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:13:22 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-16T08:13:23.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:13:22 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' 2024-09-16T08:13:23.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:13:22 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' 2024-09-16T08:13:23.683 INFO:teuthology.task.sequential:In sequential, running task cephadm.shell... 2024-09-16T08:13:23.684 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi028.front.sepia.ceph.com 2024-09-16T08:13:23.684 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5622b2fe-7402-11ef-bceb-c7b262605968 -e sha1=8293d73f8690540e843a81caec373f9cc29cf705 -- bash -c 'while ceph orch upgrade status | jq '"'"'.in_progress'"'"' | grep true && ! ceph orch upgrade status | jq '"'"'.message'"'"' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; ceph health detail ; sleep 30 ; done' 2024-09-16T08:13:23.992 INFO:teuthology.orchestra.run.smithi028.stdout: 5 11404 17.88 MB/sec warmup 22 sec latency 40.859 ms 2024-09-16T08:13:24.065 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:13:23 smithi028 ceph-mon[23744]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:8293d73f8690540e843a81caec373f9cc29cf705 2024-09-16T08:13:24.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:13:23 smithi144 ceph-mon[25668]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:8293d73f8690540e843a81caec373f9cc29cf705 2024-09-16T08:13:24.992 INFO:teuthology.orchestra.run.smithi028.stdout: 5 11711 17.37 MB/sec warmup 23 sec latency 39.448 ms 2024-09-16T08:13:25.107 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:13:24 smithi028 ceph-mon[23744]: pgmap v247: 97 pgs: 97 active+clean; 147 MiB data, 422 MiB used, 715 GiB / 715 GiB avail; 11 KiB/s rd, 8.8 MiB/s wr, 801 op/s 2024-09-16T08:13:25.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:13:24 smithi144 ceph-mon[25668]: pgmap v247: 97 pgs: 97 active+clean; 147 MiB data, 422 MiB used, 715 GiB / 715 GiB avail; 11 KiB/s rd, 8.8 MiB/s wr, 801 op/s 2024-09-16T08:13:25.992 INFO:teuthology.orchestra.run.smithi028.stdout: 5 12164 17.38 MB/sec warmup 24 sec latency 68.736 ms 2024-09-16T08:13:26.648 INFO:teuthology.orchestra.run.smithi028.stdout:true 2024-09-16T08:13:26.993 INFO:teuthology.orchestra.run.smithi028.stdout: 5 12720 16.80 MB/sec warmup 25 sec latency 51.541 ms 2024-09-16T08:13:27.148 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:13:27 smithi028 ceph-mon[23744]: pgmap v248: 97 pgs: 97 active+clean; 152 MiB data, 444 MiB used, 715 GiB / 715 GiB avail; 14 KiB/s rd, 7.1 MiB/s wr, 693 op/s 2024-09-16T08:13:27.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:13:27 smithi144 ceph-mon[25668]: pgmap v248: 97 pgs: 97 active+clean; 152 MiB data, 444 MiB used, 715 GiB / 715 GiB avail; 14 KiB/s rd, 7.1 MiB/s wr, 693 op/s 2024-09-16T08:13:27.765 INFO:teuthology.orchestra.run.smithi028.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-16T08:13:27.765 INFO:teuthology.orchestra.run.smithi028.stdout:alertmanager.smithi028 smithi028 *:9093,9094 running (5m) 28s ago 6m 24.7M - ba2b418f427c 8e1f82e9293c 2024-09-16T08:13:27.765 INFO:teuthology.orchestra.run.smithi028.stdout:crash.smithi028 smithi028 running (6m) 28s ago 6m 6405k - 17.2.0 e1d6a67b021e 69e795c000fd 2024-09-16T08:13:27.765 INFO:teuthology.orchestra.run.smithi028.stdout:crash.smithi144 smithi144 running (6m) 30s ago 6m 6400k - 17.2.0 e1d6a67b021e bbe54b51649c 2024-09-16T08:13:27.765 INFO:teuthology.orchestra.run.smithi028.stdout:grafana.smithi028 smithi028 *:3000 running (5m) 28s ago 6m 52.7M - 8.3.5 dad864ee21e9 6ff027b7efc7 2024-09-16T08:13:27.765 INFO:teuthology.orchestra.run.smithi028.stdout:mds.foofs.smithi028.srtgev smithi028 running (61s) 28s ago 60s 14.9M - 17.2.0 e1d6a67b021e 250e7a7f754c 2024-09-16T08:13:27.765 INFO:teuthology.orchestra.run.smithi028.stdout:mds.foofs.smithi144.oozgsi smithi144 running (58s) 30s ago 58s 14.4M - 17.2.0 e1d6a67b021e ecf4c2bb0fdc 2024-09-16T08:13:27.765 INFO:teuthology.orchestra.run.smithi028.stdout:mgr.smithi028.azvyzy smithi028 *:9283 running (8m) 28s ago 8m 460M - 17.2.0 e1d6a67b021e 229e5e86f252 2024-09-16T08:13:27.765 INFO:teuthology.orchestra.run.smithi028.stdout:mgr.smithi144.utzszj smithi144 *:8443,9283 running (6m) 30s ago 6m 412M - 17.2.0 e1d6a67b021e ba5c5c8ef9ca 2024-09-16T08:13:27.765 INFO:teuthology.orchestra.run.smithi028.stdout:mon.smithi028 smithi028 running (8m) 28s ago 8m 46.3M 2048M 17.2.0 e1d6a67b021e a2f688697052 2024-09-16T08:13:27.766 INFO:teuthology.orchestra.run.smithi028.stdout:mon.smithi144 smithi144 running (6m) 30s ago 6m 36.8M 2048M 17.2.0 e1d6a67b021e 1c80e3ff3507 2024-09-16T08:13:27.766 INFO:teuthology.orchestra.run.smithi028.stdout:nfs.foo.0.0.smithi028.dkttgj smithi028 *:2049 running (38s) 28s ago 38s 49.0M - 4.0 e1d6a67b021e b5dccc45a6ed 2024-09-16T08:13:27.766 INFO:teuthology.orchestra.run.smithi028.stdout:nfs.foo.1.0.smithi144.epipkg smithi144 *:2049 running (33s) 30s ago 33s 50.7M - 4.0 e1d6a67b021e faa89e61f50e 2024-09-16T08:13:27.766 INFO:teuthology.orchestra.run.smithi028.stdout:node-exporter.smithi028 smithi028 *:9100 running (6m) 28s ago 6m 17.0M - 1dbe0e931976 7ad46df29362 2024-09-16T08:13:27.766 INFO:teuthology.orchestra.run.smithi028.stdout:node-exporter.smithi144 smithi144 *:9100 running (5m) 30s ago 5m 18.6M - 1dbe0e931976 f85c6daf2fe0 2024-09-16T08:13:27.766 INFO:teuthology.orchestra.run.smithi028.stdout:osd.0 smithi028 running (5m) 28s ago 5m 43.2M 3217M 17.2.0 e1d6a67b021e 8184fbc3871f 2024-09-16T08:13:27.766 INFO:teuthology.orchestra.run.smithi028.stdout:osd.1 smithi028 running (4m) 28s ago 4m 44.3M 3217M 17.2.0 e1d6a67b021e 6911d5c68d54 2024-09-16T08:13:27.766 INFO:teuthology.orchestra.run.smithi028.stdout:osd.2 smithi028 running (4m) 28s ago 4m 39.0M 3217M 17.2.0 e1d6a67b021e 90b27d8375a2 2024-09-16T08:13:27.766 INFO:teuthology.orchestra.run.smithi028.stdout:osd.3 smithi028 running (4m) 28s ago 4m 42.6M 3217M 17.2.0 e1d6a67b021e dac477d8b04f 2024-09-16T08:13:27.766 INFO:teuthology.orchestra.run.smithi028.stdout:osd.4 smithi144 running (3m) 30s ago 3m 40.0M 3984M 17.2.0 e1d6a67b021e 86719c60975c 2024-09-16T08:13:27.766 INFO:teuthology.orchestra.run.smithi028.stdout:osd.5 smithi144 running (3m) 30s ago 3m 38.7M 3984M 17.2.0 e1d6a67b021e 70452c145c60 2024-09-16T08:13:27.766 INFO:teuthology.orchestra.run.smithi028.stdout:osd.6 smithi144 running (2m) 30s ago 2m 40.2M 3984M 17.2.0 e1d6a67b021e 7f9981be792f 2024-09-16T08:13:27.766 INFO:teuthology.orchestra.run.smithi028.stdout:osd.7 smithi144 running (2m) 30s ago 2m 41.8M 3984M 17.2.0 e1d6a67b021e dada74ddbe4c 2024-09-16T08:13:27.767 INFO:teuthology.orchestra.run.smithi028.stdout:prometheus.smithi028 smithi028 *:9095 running (5m) 28s ago 5m 63.2M - 514e6a882f6e fd3df637374d 2024-09-16T08:13:27.993 INFO:teuthology.orchestra.run.smithi028.stdout: 5 13197 16.63 MB/sec warmup 26 sec latency 25.967 ms 2024-09-16T08:13:28.156 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:13:28 smithi028 ceph-mon[23744]: from='client.24393 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T08:13:28.157 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:13:28 smithi028 ceph-mon[23744]: from='client.24395 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T08:13:28.407 INFO:teuthology.orchestra.run.smithi028.stdout:{ 2024-09-16T08:13:28.407 INFO:teuthology.orchestra.run.smithi028.stdout: "mon": { 2024-09-16T08:13:28.408 INFO:teuthology.orchestra.run.smithi028.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-16T08:13:28.408 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-09-16T08:13:28.408 INFO:teuthology.orchestra.run.smithi028.stdout: "mgr": { 2024-09-16T08:13:28.408 INFO:teuthology.orchestra.run.smithi028.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-16T08:13:28.408 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-09-16T08:13:28.408 INFO:teuthology.orchestra.run.smithi028.stdout: "osd": { 2024-09-16T08:13:28.408 INFO:teuthology.orchestra.run.smithi028.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-09-16T08:13:28.408 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-09-16T08:13:28.408 INFO:teuthology.orchestra.run.smithi028.stdout: "mds": { 2024-09-16T08:13:28.408 INFO:teuthology.orchestra.run.smithi028.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-16T08:13:28.408 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-09-16T08:13:28.408 INFO:teuthology.orchestra.run.smithi028.stdout: "overall": { 2024-09-16T08:13:28.409 INFO:teuthology.orchestra.run.smithi028.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 14 2024-09-16T08:13:28.409 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-09-16T08:13:28.409 INFO:teuthology.orchestra.run.smithi028.stdout:} 2024-09-16T08:13:28.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:13:28 smithi144 ceph-mon[25668]: from='client.24393 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T08:13:28.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:13:28 smithi144 ceph-mon[25668]: from='client.24395 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T08:13:28.970 INFO:teuthology.orchestra.run.smithi028.stdout:{ 2024-09-16T08:13:28.970 INFO:teuthology.orchestra.run.smithi028.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph:8293d73f8690540e843a81caec373f9cc29cf705", 2024-09-16T08:13:28.971 INFO:teuthology.orchestra.run.smithi028.stdout: "in_progress": true, 2024-09-16T08:13:28.971 INFO:teuthology.orchestra.run.smithi028.stdout: "services_complete": [], 2024-09-16T08:13:28.971 INFO:teuthology.orchestra.run.smithi028.stdout: "progress": "", 2024-09-16T08:13:28.971 INFO:teuthology.orchestra.run.smithi028.stdout: "message": "Doing first pull of quay.ceph.io/ceph-ci/ceph:8293d73f8690540e843a81caec373f9cc29cf705 image" 2024-09-16T08:13:28.971 INFO:teuthology.orchestra.run.smithi028.stdout:} 2024-09-16T08:13:28.993 INFO:teuthology.orchestra.run.smithi028.stdout: 5 13874 17.02 MB/sec warmup 27 sec latency 20.751 ms 2024-09-16T08:13:29.174 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:13:29 smithi028 ceph-mon[23744]: pgmap v249: 97 pgs: 97 active+clean; 152 MiB data, 459 MiB used, 715 GiB / 715 GiB avail; 7.7 KiB/s rd, 7.6 MiB/s wr, 738 op/s 2024-09-16T08:13:29.174 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:13:29 smithi028 ceph-mon[23744]: from='client.14620 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T08:13:29.174 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:13:29 smithi028 ceph-mon[23744]: from='client.? 172.21.15.28:0/2021858130' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-16T08:13:29.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:13:29 smithi144 ceph-mon[25668]: pgmap v249: 97 pgs: 97 active+clean; 152 MiB data, 459 MiB used, 715 GiB / 715 GiB avail; 7.7 KiB/s rd, 7.6 MiB/s wr, 738 op/s 2024-09-16T08:13:29.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:13:29 smithi144 ceph-mon[25668]: from='client.14620 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T08:13:29.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:13:29 smithi144 ceph-mon[25668]: from='client.? 172.21.15.28:0/2021858130' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-16T08:13:29.655 INFO:teuthology.orchestra.run.smithi028.stdout:HEALTH_OK 2024-09-16T08:13:29.993 INFO:teuthology.orchestra.run.smithi028.stdout: 5 14412 17.27 MB/sec warmup 28 sec latency 18.152 ms 2024-09-16T08:13:30.182 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:13:30 smithi028 ceph-mon[23744]: from='client.14628 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T08:13:30.182 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:13:30 smithi028 ceph-mon[23744]: from='client.? 172.21.15.28:0/1478915444' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-16T08:13:30.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:13:30 smithi144 ceph-mon[25668]: from='client.14628 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T08:13:30.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:13:30 smithi144 ceph-mon[25668]: from='client.? 172.21.15.28:0/1478915444' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-16T08:13:30.993 INFO:teuthology.orchestra.run.smithi028.stdout: 5 14828 16.82 MB/sec warmup 29 sec latency 22.332 ms 2024-09-16T08:13:31.186 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:13:31 smithi028 ceph-mon[23744]: pgmap v250: 97 pgs: 97 active+clean; 152 MiB data, 459 MiB used, 715 GiB / 715 GiB avail; 7.5 KiB/s rd, 5.5 MiB/s wr, 567 op/s 2024-09-16T08:13:31.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:13:31 smithi144 ceph-mon[25668]: pgmap v250: 97 pgs: 97 active+clean; 152 MiB data, 459 MiB used, 715 GiB / 715 GiB avail; 7.5 KiB/s rd, 5.5 MiB/s wr, 567 op/s 2024-09-16T08:13:31.993 INFO:teuthology.orchestra.run.smithi028.stdout: 5 15490 16.78 MB/sec warmup 30 sec latency 14.768 ms 2024-09-16T08:13:32.993 INFO:teuthology.orchestra.run.smithi028.stdout: 5 16250 16.62 MB/sec warmup 31 sec latency 56.257 ms 2024-09-16T08:13:33.226 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:13:33 smithi028 ceph-mon[23744]: pgmap v251: 97 pgs: 97 active+clean; 156 MiB data, 484 MiB used, 715 GiB / 715 GiB avail; 12 KiB/s rd, 6.9 MiB/s wr, 715 op/s 2024-09-16T08:13:33.449 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:13:33 smithi144 ceph-mon[25668]: pgmap v251: 97 pgs: 97 active+clean; 156 MiB data, 484 MiB used, 715 GiB / 715 GiB avail; 12 KiB/s rd, 6.9 MiB/s wr, 715 op/s 2024-09-16T08:13:33.993 INFO:teuthology.orchestra.run.smithi028.stdout: 5 16705 16.41 MB/sec warmup 32 sec latency 74.359 ms 2024-09-16T08:13:34.993 INFO:teuthology.orchestra.run.smithi028.stdout: 5 17663 17.14 MB/sec warmup 33 sec latency 17.663 ms 2024-09-16T08:13:35.237 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:13:35 smithi028 ceph-mon[23744]: pgmap v252: 97 pgs: 97 active+clean; 172 MiB data, 554 MiB used, 715 GiB / 715 GiB avail; 7.2 KiB/s rd, 7.9 MiB/s wr, 705 op/s 2024-09-16T08:13:35.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:13:35 smithi144 ceph-mon[25668]: pgmap v252: 97 pgs: 97 active+clean; 172 MiB data, 554 MiB used, 715 GiB / 715 GiB avail; 7.2 KiB/s rd, 7.9 MiB/s wr, 705 op/s 2024-09-16T08:13:35.993 INFO:teuthology.orchestra.run.smithi028.stdout: 5 18196 17.14 MB/sec warmup 34 sec latency 16.996 ms 2024-09-16T08:13:36.993 INFO:teuthology.orchestra.run.smithi028.stdout: 5 18839 16.88 MB/sec warmup 35 sec latency 31.475 ms 2024-09-16T08:13:37.249 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:13:37 smithi028 ceph-mon[23744]: pgmap v253: 97 pgs: 97 active+clean; 178 MiB data, 575 MiB used, 715 GiB / 715 GiB avail; 13 KiB/s rd, 7.5 MiB/s wr, 746 op/s 2024-09-16T08:13:37.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:13:37 smithi144 ceph-mon[25668]: pgmap v253: 97 pgs: 97 active+clean; 178 MiB data, 575 MiB used, 715 GiB / 715 GiB avail; 13 KiB/s rd, 7.5 MiB/s wr, 746 op/s 2024-09-16T08:13:37.993 INFO:teuthology.orchestra.run.smithi028.stdout: 5 19288 16.90 MB/sec warmup 36 sec latency 39.473 ms 2024-09-16T08:13:38.993 INFO:teuthology.orchestra.run.smithi028.stdout: 5 20210 16.73 MB/sec warmup 37 sec latency 20.574 ms 2024-09-16T08:13:39.262 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:13:39 smithi028 ceph-mon[23744]: pgmap v254: 97 pgs: 97 active+clean; 182 MiB data, 628 MiB used, 715 GiB / 715 GiB avail; 10 KiB/s rd, 9.5 MiB/s wr, 922 op/s 2024-09-16T08:13:39.585 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:13:39 smithi144 ceph-mon[25668]: pgmap v254: 97 pgs: 97 active+clean; 182 MiB data, 628 MiB used, 715 GiB / 715 GiB avail; 10 KiB/s rd, 9.5 MiB/s wr, 922 op/s 2024-09-16T08:13:39.993 INFO:teuthology.orchestra.run.smithi028.stdout: 5 21253 17.48 MB/sec warmup 38 sec latency 11.285 ms 2024-09-16T08:13:40.994 INFO:teuthology.orchestra.run.smithi028.stdout: 5 21797 17.40 MB/sec warmup 39 sec latency 14.728 ms 2024-09-16T08:13:41.153 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:13:41 smithi028 ceph-mon[23744]: pgmap v255: 97 pgs: 97 active+clean; 182 MiB data, 628 MiB used, 715 GiB / 715 GiB avail; 10 KiB/s rd, 7.8 MiB/s wr, 724 op/s 2024-09-16T08:13:41.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:13:41 smithi144 ceph-mon[25668]: pgmap v255: 97 pgs: 97 active+clean; 182 MiB data, 628 MiB used, 715 GiB / 715 GiB avail; 10 KiB/s rd, 7.8 MiB/s wr, 724 op/s 2024-09-16T08:13:41.993 INFO:teuthology.orchestra.run.smithi028.stdout: 5 22526 17.34 MB/sec warmup 40 sec latency 11.751 ms 2024-09-16T08:13:42.993 INFO:teuthology.orchestra.run.smithi028.stdout: 5 23422 17.29 MB/sec warmup 41 sec latency 40.358 ms 2024-09-16T08:13:43.162 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:13:43 smithi028 ceph-mon[23744]: pgmap v256: 97 pgs: 97 active+clean; 182 MiB data, 637 MiB used, 715 GiB / 715 GiB avail; 15 KiB/s rd, 9.3 MiB/s wr, 797 op/s 2024-09-16T08:13:43.441 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:13:43 smithi144 ceph-mon[25668]: pgmap v256: 97 pgs: 97 active+clean; 182 MiB data, 637 MiB used, 715 GiB / 715 GiB avail; 15 KiB/s rd, 9.3 MiB/s wr, 797 op/s 2024-09-16T08:13:43.994 INFO:teuthology.orchestra.run.smithi028.stdout: 5 23950 17.22 MB/sec warmup 42 sec latency 46.149 ms 2024-09-16T08:13:44.994 INFO:teuthology.orchestra.run.smithi028.stdout: 5 24867 17.69 MB/sec warmup 43 sec latency 14.198 ms 2024-09-16T08:13:45.178 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:13:45 smithi028 ceph-mon[23744]: pgmap v257: 97 pgs: 97 active+clean; 189 MiB data, 686 MiB used, 715 GiB / 715 GiB avail; 11 KiB/s rd, 10 MiB/s wr, 873 op/s 2024-09-16T08:13:45.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:13:45 smithi144 ceph-mon[25668]: pgmap v257: 97 pgs: 97 active+clean; 189 MiB data, 686 MiB used, 715 GiB / 715 GiB avail; 11 KiB/s rd, 10 MiB/s wr, 873 op/s 2024-09-16T08:13:45.993 INFO:teuthology.orchestra.run.smithi028.stdout: 5 25476 17.61 MB/sec warmup 44 sec latency 20.193 ms 2024-09-16T08:13:46.994 INFO:teuthology.orchestra.run.smithi028.stdout: 5 26178 17.57 MB/sec warmup 45 sec latency 31.205 ms 2024-09-16T08:13:47.188 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:13:47 smithi028 ceph-mon[23744]: pgmap v258: 97 pgs: 97 active+clean; 202 MiB data, 756 MiB used, 714 GiB / 715 GiB avail; 17 KiB/s rd, 9.8 MiB/s wr, 941 op/s 2024-09-16T08:13:47.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:13:47 smithi144 ceph-mon[25668]: pgmap v258: 97 pgs: 97 active+clean; 202 MiB data, 756 MiB used, 714 GiB / 715 GiB avail; 17 KiB/s rd, 9.8 MiB/s wr, 941 op/s 2024-09-16T08:13:47.994 INFO:teuthology.orchestra.run.smithi028.stdout: 5 26871 17.43 MB/sec warmup 46 sec latency 25.716 ms 2024-09-16T08:13:48.994 INFO:teuthology.orchestra.run.smithi028.stdout: 5 27543 17.48 MB/sec warmup 47 sec latency 27.530 ms 2024-09-16T08:13:49.195 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:13:49 smithi028 ceph-mon[23744]: pgmap v259: 97 pgs: 97 active+clean; 206 MiB data, 798 MiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 11 MiB/s wr, 1.02k op/s 2024-09-16T08:13:49.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:13:49 smithi144 ceph-mon[25668]: pgmap v259: 97 pgs: 97 active+clean; 206 MiB data, 798 MiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 11 MiB/s wr, 1.02k op/s 2024-09-16T08:13:49.994 INFO:teuthology.orchestra.run.smithi028.stdout: 5 28576 18.02 MB/sec warmup 48 sec latency 21.579 ms 2024-09-16T08:13:50.994 INFO:teuthology.orchestra.run.smithi028.stdout: 5 29231 17.80 MB/sec warmup 49 sec latency 17.142 ms 2024-09-16T08:13:51.204 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:13:51 smithi028 ceph-mon[23744]: pgmap v260: 97 pgs: 97 active+clean; 206 MiB data, 798 MiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 8.6 MiB/s wr, 784 op/s 2024-09-16T08:13:51.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:13:51 smithi144 ceph-mon[25668]: pgmap v260: 97 pgs: 97 active+clean; 206 MiB data, 798 MiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 8.6 MiB/s wr, 784 op/s 2024-09-16T08:13:51.994 INFO:teuthology.orchestra.run.smithi028.stdout: 5 30067 17.92 MB/sec warmup 50 sec latency 13.296 ms 2024-09-16T08:13:52.994 INFO:teuthology.orchestra.run.smithi028.stdout: 5 30870 17.78 MB/sec warmup 51 sec latency 22.676 ms 2024-09-16T08:13:53.221 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:13:53 smithi028 ceph-mon[23744]: pgmap v261: 97 pgs: 97 active+clean; 210 MiB data, 843 MiB used, 714 GiB / 715 GiB avail; 17 KiB/s rd, 10 MiB/s wr, 927 op/s 2024-09-16T08:13:53.441 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:13:53 smithi144 ceph-mon[25668]: pgmap v261: 97 pgs: 97 active+clean; 210 MiB data, 843 MiB used, 714 GiB / 715 GiB avail; 17 KiB/s rd, 10 MiB/s wr, 927 op/s 2024-09-16T08:13:53.994 INFO:teuthology.orchestra.run.smithi028.stdout: 5 31734 18.16 MB/sec warmup 52 sec latency 21.731 ms 2024-09-16T08:13:54.994 INFO:teuthology.orchestra.run.smithi028.stdout: 5 32319 18.20 MB/sec warmup 53 sec latency 21.602 ms 2024-09-16T08:13:55.197 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:13:55 smithi028 ceph-mon[23744]: pgmap v262: 97 pgs: 97 active+clean; 212 MiB data, 883 MiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 11 MiB/s wr, 1.02k op/s 2024-09-16T08:13:55.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:13:55 smithi144 ceph-mon[25668]: pgmap v262: 97 pgs: 97 active+clean; 212 MiB data, 883 MiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 11 MiB/s wr, 1.02k op/s 2024-09-16T08:13:55.994 INFO:teuthology.orchestra.run.smithi028.stdout: 5 32801 17.94 MB/sec warmup 54 sec latency 40.698 ms 2024-09-16T08:13:56.994 INFO:teuthology.orchestra.run.smithi028.stdout: 5 33301 17.85 MB/sec warmup 55 sec latency 32.668 ms 2024-09-16T08:13:57.241 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:13:57 smithi028 ceph-mon[23744]: pgmap v263: 97 pgs: 97 active+clean; 222 MiB data, 917 MiB used, 714 GiB / 715 GiB avail; 18 KiB/s rd, 10 MiB/s wr, 1.02k op/s 2024-09-16T08:13:57.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:13:57 smithi144 ceph-mon[25668]: pgmap v263: 97 pgs: 97 active+clean; 222 MiB data, 917 MiB used, 714 GiB / 715 GiB avail; 18 KiB/s rd, 10 MiB/s wr, 1.02k op/s 2024-09-16T08:13:57.994 INFO:teuthology.orchestra.run.smithi028.stdout: 5 33719 17.71 MB/sec warmup 56 sec latency 56.667 ms 2024-09-16T08:13:58.994 INFO:teuthology.orchestra.run.smithi028.stdout: 5 34356 17.56 MB/sec warmup 57 sec latency 46.227 ms 2024-09-16T08:13:59.218 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:13:59 smithi028 ceph-mon[23744]: pgmap v264: 97 pgs: 97 active+clean; 229 MiB data, 964 MiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 11 MiB/s wr, 993 op/s 2024-09-16T08:13:59.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:13:59 smithi144 ceph-mon[25668]: pgmap v264: 97 pgs: 97 active+clean; 229 MiB data, 964 MiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 11 MiB/s wr, 993 op/s 2024-09-16T08:13:59.994 INFO:teuthology.orchestra.run.smithi028.stdout: 5 35379 17.99 MB/sec warmup 58 sec latency 26.538 ms 2024-09-16T08:14:00.234 INFO:teuthology.orchestra.run.smithi028.stdout:true 2024-09-16T08:14:00.994 INFO:teuthology.orchestra.run.smithi028.stdout: 5 35942 17.97 MB/sec warmup 59 sec latency 40.479 ms 2024-09-16T08:14:01.273 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:14:01 smithi028 ceph-mon[23744]: pgmap v265: 97 pgs: 97 active+clean; 229 MiB data, 964 MiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 8.6 MiB/s wr, 785 op/s 2024-09-16T08:14:01.273 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:14:01 smithi028 ceph-mon[23744]: from='client.24415 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T08:14:01.350 INFO:teuthology.orchestra.run.smithi028.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-16T08:14:01.350 INFO:teuthology.orchestra.run.smithi028.stdout:alertmanager.smithi028 smithi028 *:9093,9094 running (6m) 61s ago 7m 24.7M - ba2b418f427c 8e1f82e9293c 2024-09-16T08:14:01.350 INFO:teuthology.orchestra.run.smithi028.stdout:crash.smithi028 smithi028 running (7m) 61s ago 7m 6405k - 17.2.0 e1d6a67b021e 69e795c000fd 2024-09-16T08:14:01.351 INFO:teuthology.orchestra.run.smithi028.stdout:crash.smithi144 smithi144 running (6m) 63s ago 6m 6400k - 17.2.0 e1d6a67b021e bbe54b51649c 2024-09-16T08:14:01.351 INFO:teuthology.orchestra.run.smithi028.stdout:grafana.smithi028 smithi028 *:3000 running (5m) 61s ago 7m 52.7M - 8.3.5 dad864ee21e9 6ff027b7efc7 2024-09-16T08:14:01.351 INFO:teuthology.orchestra.run.smithi028.stdout:mds.foofs.smithi028.srtgev smithi028 running (94s) 61s ago 94s 14.9M - 17.2.0 e1d6a67b021e 250e7a7f754c 2024-09-16T08:14:01.351 INFO:teuthology.orchestra.run.smithi028.stdout:mds.foofs.smithi144.oozgsi smithi144 running (92s) 63s ago 91s 14.4M - 17.2.0 e1d6a67b021e ecf4c2bb0fdc 2024-09-16T08:14:01.351 INFO:teuthology.orchestra.run.smithi028.stdout:mgr.smithi028.azvyzy smithi028 *:9283 running (8m) 61s ago 8m 460M - 17.2.0 e1d6a67b021e 229e5e86f252 2024-09-16T08:14:01.351 INFO:teuthology.orchestra.run.smithi028.stdout:mgr.smithi144.utzszj smithi144 *:8443,9283 running (6m) 63s ago 6m 412M - 17.2.0 e1d6a67b021e ba5c5c8ef9ca 2024-09-16T08:14:01.351 INFO:teuthology.orchestra.run.smithi028.stdout:mon.smithi028 smithi028 running (8m) 61s ago 8m 46.3M 2048M 17.2.0 e1d6a67b021e a2f688697052 2024-09-16T08:14:01.351 INFO:teuthology.orchestra.run.smithi028.stdout:mon.smithi144 smithi144 running (6m) 63s ago 6m 36.8M 2048M 17.2.0 e1d6a67b021e 1c80e3ff3507 2024-09-16T08:14:01.351 INFO:teuthology.orchestra.run.smithi028.stdout:nfs.foo.0.0.smithi028.dkttgj smithi028 *:2049 running (72s) 61s ago 71s 49.0M - 4.0 e1d6a67b021e b5dccc45a6ed 2024-09-16T08:14:01.351 INFO:teuthology.orchestra.run.smithi028.stdout:nfs.foo.1.0.smithi144.epipkg smithi144 *:2049 running (67s) 63s ago 66s 50.7M - 4.0 e1d6a67b021e faa89e61f50e 2024-09-16T08:14:01.351 INFO:teuthology.orchestra.run.smithi028.stdout:node-exporter.smithi028 smithi028 *:9100 running (7m) 61s ago 7m 17.0M - 1dbe0e931976 7ad46df29362 2024-09-16T08:14:01.351 INFO:teuthology.orchestra.run.smithi028.stdout:node-exporter.smithi144 smithi144 *:9100 running (6m) 63s ago 6m 18.6M - 1dbe0e931976 f85c6daf2fe0 2024-09-16T08:14:01.351 INFO:teuthology.orchestra.run.smithi028.stdout:osd.0 smithi028 running (5m) 61s ago 5m 43.2M 3217M 17.2.0 e1d6a67b021e 8184fbc3871f 2024-09-16T08:14:01.352 INFO:teuthology.orchestra.run.smithi028.stdout:osd.1 smithi028 running (5m) 61s ago 5m 44.3M 3217M 17.2.0 e1d6a67b021e 6911d5c68d54 2024-09-16T08:14:01.352 INFO:teuthology.orchestra.run.smithi028.stdout:osd.2 smithi028 running (5m) 61s ago 5m 39.0M 3217M 17.2.0 e1d6a67b021e 90b27d8375a2 2024-09-16T08:14:01.352 INFO:teuthology.orchestra.run.smithi028.stdout:osd.3 smithi028 running (4m) 61s ago 4m 42.6M 3217M 17.2.0 e1d6a67b021e dac477d8b04f 2024-09-16T08:14:01.352 INFO:teuthology.orchestra.run.smithi028.stdout:osd.4 smithi144 running (4m) 63s ago 4m 40.0M 3984M 17.2.0 e1d6a67b021e 86719c60975c 2024-09-16T08:14:01.352 INFO:teuthology.orchestra.run.smithi028.stdout:osd.5 smithi144 running (3m) 63s ago 3m 38.7M 3984M 17.2.0 e1d6a67b021e 70452c145c60 2024-09-16T08:14:01.352 INFO:teuthology.orchestra.run.smithi028.stdout:osd.6 smithi144 running (3m) 63s ago 3m 40.2M 3984M 17.2.0 e1d6a67b021e 7f9981be792f 2024-09-16T08:14:01.352 INFO:teuthology.orchestra.run.smithi028.stdout:osd.7 smithi144 running (3m) 63s ago 3m 41.8M 3984M 17.2.0 e1d6a67b021e dada74ddbe4c 2024-09-16T08:14:01.352 INFO:teuthology.orchestra.run.smithi028.stdout:prometheus.smithi028 smithi028 *:9095 running (6m) 61s ago 6m 63.2M - 514e6a882f6e fd3df637374d 2024-09-16T08:14:01.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:14:01 smithi144 ceph-mon[25668]: pgmap v265: 97 pgs: 97 active+clean; 229 MiB data, 964 MiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 8.6 MiB/s wr, 785 op/s 2024-09-16T08:14:01.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:14:01 smithi144 ceph-mon[25668]: from='client.24415 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T08:14:01.994 INFO:teuthology.orchestra.run.smithi028.stdout: 5 36362 17.74 MB/sec warmup 60 sec latency 27.769 ms 2024-09-16T08:14:02.044 INFO:teuthology.orchestra.run.smithi028.stdout:{ 2024-09-16T08:14:02.044 INFO:teuthology.orchestra.run.smithi028.stdout: "mon": { 2024-09-16T08:14:02.045 INFO:teuthology.orchestra.run.smithi028.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-16T08:14:02.045 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-09-16T08:14:02.045 INFO:teuthology.orchestra.run.smithi028.stdout: "mgr": { 2024-09-16T08:14:02.045 INFO:teuthology.orchestra.run.smithi028.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-16T08:14:02.045 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-09-16T08:14:02.045 INFO:teuthology.orchestra.run.smithi028.stdout: "osd": { 2024-09-16T08:14:02.045 INFO:teuthology.orchestra.run.smithi028.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-09-16T08:14:02.045 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-09-16T08:14:02.045 INFO:teuthology.orchestra.run.smithi028.stdout: "mds": { 2024-09-16T08:14:02.045 INFO:teuthology.orchestra.run.smithi028.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-16T08:14:02.045 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-09-16T08:14:02.045 INFO:teuthology.orchestra.run.smithi028.stdout: "overall": { 2024-09-16T08:14:02.046 INFO:teuthology.orchestra.run.smithi028.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 14 2024-09-16T08:14:02.046 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-09-16T08:14:02.046 INFO:teuthology.orchestra.run.smithi028.stdout:} 2024-09-16T08:14:02.279 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:14:02 smithi028 ceph-mon[23744]: from='client.24419 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T08:14:02.279 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:14:02 smithi028 ceph-mon[23744]: from='client.? 172.21.15.28:0/1521465274' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-16T08:14:02.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:14:02 smithi144 ceph-mon[25668]: from='client.24419 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T08:14:02.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:14:02 smithi144 ceph-mon[25668]: from='client.? 172.21.15.28:0/1521465274' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-16T08:14:02.605 INFO:teuthology.orchestra.run.smithi028.stdout:{ 2024-09-16T08:14:02.606 INFO:teuthology.orchestra.run.smithi028.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph:8293d73f8690540e843a81caec373f9cc29cf705", 2024-09-16T08:14:02.606 INFO:teuthology.orchestra.run.smithi028.stdout: "in_progress": true, 2024-09-16T08:14:02.606 INFO:teuthology.orchestra.run.smithi028.stdout: "services_complete": [], 2024-09-16T08:14:02.606 INFO:teuthology.orchestra.run.smithi028.stdout: "progress": "", 2024-09-16T08:14:02.606 INFO:teuthology.orchestra.run.smithi028.stdout: "message": "Doing first pull of quay.ceph.io/ceph-ci/ceph:8293d73f8690540e843a81caec373f9cc29cf705 image" 2024-09-16T08:14:02.606 INFO:teuthology.orchestra.run.smithi028.stdout:} 2024-09-16T08:14:02.994 INFO:teuthology.orchestra.run.smithi028.stdout: 5 36805 17.66 MB/sec warmup 61 sec latency 23.687 ms 2024-09-16T08:14:03.264 INFO:teuthology.orchestra.run.smithi028.stdout:HEALTH_OK 2024-09-16T08:14:03.287 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:14:03 smithi028 ceph-mon[23744]: from='client.14644 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T08:14:03.287 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:14:03 smithi028 ceph-mon[23744]: pgmap v266: 97 pgs: 97 active+clean; 232 MiB data, 1004 MiB used, 714 GiB / 715 GiB avail; 17 KiB/s rd, 10 MiB/s wr, 948 op/s 2024-09-16T08:14:03.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:14:03 smithi144 ceph-mon[25668]: from='client.14644 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T08:14:03.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:14:03 smithi144 ceph-mon[25668]: pgmap v266: 97 pgs: 97 active+clean; 232 MiB data, 1004 MiB used, 714 GiB / 715 GiB avail; 17 KiB/s rd, 10 MiB/s wr, 948 op/s 2024-09-16T08:14:03.994 INFO:teuthology.orchestra.run.smithi028.stdout: 5 37385 17.56 MB/sec warmup 62 sec latency 30.589 ms 2024-09-16T08:14:04.293 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:14:04 smithi028 ceph-mon[23744]: from='client.14652 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T08:14:04.293 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:14:04 smithi028 ceph-mon[23744]: from='client.? 172.21.15.28:0/2781341581' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-16T08:14:04.293 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:14:04 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi028.azvyzy/mirror_snapshot_schedule"}]: dispatch 2024-09-16T08:14:04.293 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:14:04 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi028.azvyzy/trash_purge_schedule"}]: dispatch 2024-09-16T08:14:04.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:14:04 smithi144 ceph-mon[25668]: from='client.14652 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T08:14:04.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:14:04 smithi144 ceph-mon[25668]: from='client.? 172.21.15.28:0/2781341581' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-16T08:14:04.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:14:04 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi028.azvyzy/mirror_snapshot_schedule"}]: dispatch 2024-09-16T08:14:04.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:14:04 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi028.azvyzy/trash_purge_schedule"}]: dispatch 2024-09-16T08:14:04.994 INFO:teuthology.orchestra.run.smithi028.stdout: 5 38014 17.48 MB/sec warmup 63 sec latency 32.352 ms 2024-09-16T08:14:05.296 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:14:05 smithi028 ceph-mon[23744]: pgmap v267: 97 pgs: 97 active+clean; 233 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 10 MiB/s wr, 901 op/s 2024-09-16T08:14:05.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:14:05 smithi144 ceph-mon[25668]: pgmap v267: 97 pgs: 97 active+clean; 233 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 10 MiB/s wr, 901 op/s 2024-09-16T08:14:05.995 INFO:teuthology.orchestra.run.smithi028.stdout: 5 38625 17.64 MB/sec warmup 64 sec latency 22.105 ms 2024-09-16T08:14:06.994 INFO:teuthology.orchestra.run.smithi028.stdout: 5 39141 17.66 MB/sec warmup 65 sec latency 24.291 ms 2024-09-16T08:14:07.315 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:14:07 smithi028 ceph-mon[23744]: pgmap v268: 97 pgs: 97 active+clean; 241 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 17 KiB/s rd, 9.5 MiB/s wr, 957 op/s 2024-09-16T08:14:07.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:14:07 smithi144 ceph-mon[25668]: pgmap v268: 97 pgs: 97 active+clean; 241 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 17 KiB/s rd, 9.5 MiB/s wr, 957 op/s 2024-09-16T08:14:07.995 INFO:teuthology.orchestra.run.smithi028.stdout: 5 39423 17.51 MB/sec warmup 66 sec latency 83.319 ms 2024-09-16T08:14:08.995 INFO:teuthology.orchestra.run.smithi028.stdout: 5 39717 17.30 MB/sec warmup 67 sec latency 46.058 ms 2024-09-16T08:14:09.356 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:14:09 smithi028 ceph-mon[23744]: pgmap v269: 97 pgs: 97 active+clean; 240 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 9.8 MiB/s wr, 851 op/s 2024-09-16T08:14:09.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:14:09 smithi144 ceph-mon[25668]: pgmap v269: 97 pgs: 97 active+clean; 240 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 9.8 MiB/s wr, 851 op/s 2024-09-16T08:14:09.995 INFO:teuthology.orchestra.run.smithi028.stdout: 5 40269 17.20 MB/sec warmup 68 sec latency 30.811 ms 2024-09-16T08:14:10.995 INFO:teuthology.orchestra.run.smithi028.stdout: 5 40972 17.19 MB/sec warmup 69 sec latency 29.490 ms 2024-09-16T08:14:11.366 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:14:11 smithi028 ceph-mon[23744]: pgmap v270: 97 pgs: 97 active+clean; 240 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 6.7 MiB/s wr, 597 op/s 2024-09-16T08:14:11.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:14:11 smithi144 ceph-mon[25668]: pgmap v270: 97 pgs: 97 active+clean; 240 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 6.7 MiB/s wr, 597 op/s 2024-09-16T08:14:11.995 INFO:teuthology.orchestra.run.smithi028.stdout: 5 41753 17.21 MB/sec warmup 70 sec latency 15.594 ms 2024-09-16T08:14:12.995 INFO:teuthology.orchestra.run.smithi028.stdout: 5 42352 17.34 MB/sec warmup 71 sec latency 63.376 ms 2024-09-16T08:14:13.391 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:14:13 smithi028 ceph-mon[23744]: pgmap v271: 97 pgs: 97 active+clean; 248 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 14 KiB/s rd, 8.1 MiB/s wr, 747 op/s 2024-09-16T08:14:13.392 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:14:13 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' 2024-09-16T08:14:13.392 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:14:13 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-16T08:14:13.392 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:14:13 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' 2024-09-16T08:14:13.519 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:14:13 smithi144 ceph-mon[25668]: pgmap v271: 97 pgs: 97 active+clean; 248 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 14 KiB/s rd, 8.1 MiB/s wr, 747 op/s 2024-09-16T08:14:13.519 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:14:13 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' 2024-09-16T08:14:13.519 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:14:13 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-16T08:14:13.519 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:14:13 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' 2024-09-16T08:14:13.995 INFO:teuthology.orchestra.run.smithi028.stdout: 5 42631 17.19 MB/sec warmup 72 sec latency 53.584 ms 2024-09-16T08:14:14.401 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:14:14 smithi028 ceph-mon[23744]: Upgrade: Target is version 19.3.0-4917-g8293d73f (unknown) 2024-09-16T08:14:14.401 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:14:14 smithi028 ceph-mon[23744]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:f7ad55f9ccd17ba66aadb19b4d291e67b60ea24e5d8a0c0e92941fd59c81e4ce, digests ['quay.ceph.io/ceph-ci/ceph@sha256:f7ad55f9ccd17ba66aadb19b4d291e67b60ea24e5d8a0c0e92941fd59c81e4ce'] 2024-09-16T08:14:14.402 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:14:14 smithi028 ceph-mon[23744]: Upgrade: Need to upgrade myself (mgr.smithi028.azvyzy) 2024-09-16T08:14:14.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:14:14 smithi144 ceph-mon[25668]: Upgrade: Target is version 19.3.0-4917-g8293d73f (unknown) 2024-09-16T08:14:14.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:14:14 smithi144 ceph-mon[25668]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:f7ad55f9ccd17ba66aadb19b4d291e67b60ea24e5d8a0c0e92941fd59c81e4ce, digests ['quay.ceph.io/ceph-ci/ceph@sha256:f7ad55f9ccd17ba66aadb19b4d291e67b60ea24e5d8a0c0e92941fd59c81e4ce'] 2024-09-16T08:14:14.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:14:14 smithi144 ceph-mon[25668]: Upgrade: Need to upgrade myself (mgr.smithi028.azvyzy) 2024-09-16T08:14:14.995 INFO:teuthology.orchestra.run.smithi028.stdout: 5 42924 17.11 MB/sec warmup 73 sec latency 141.982 ms 2024-09-16T08:14:15.412 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:14:15 smithi028 ceph-mon[23744]: Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:f7ad55f9ccd17ba66aadb19b4d291e67b60ea24e5d8a0c0e92941fd59c81e4ce on smithi144 2024-09-16T08:14:15.412 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:14:15 smithi028 ceph-mon[23744]: pgmap v272: 97 pgs: 97 active+clean; 249 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 8.4 KiB/s rd, 8.2 MiB/s wr, 769 op/s 2024-09-16T08:14:15.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:14:15 smithi144 ceph-mon[25668]: Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:f7ad55f9ccd17ba66aadb19b4d291e67b60ea24e5d8a0c0e92941fd59c81e4ce on smithi144 2024-09-16T08:14:15.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:14:15 smithi144 ceph-mon[25668]: pgmap v272: 97 pgs: 97 active+clean; 249 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 8.4 KiB/s rd, 8.2 MiB/s wr, 769 op/s 2024-09-16T08:14:15.995 INFO:teuthology.orchestra.run.smithi028.stdout: 5 43129 16.93 MB/sec warmup 74 sec latency 60.249 ms 2024-09-16T08:14:16.996 INFO:teuthology.orchestra.run.smithi028.stdout: 5 43358 16.75 MB/sec warmup 75 sec latency 99.853 ms 2024-09-16T08:14:17.245 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:14:17 smithi028 ceph-mon[23744]: pgmap v273: 97 pgs: 97 active+clean; 252 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 13 KiB/s rd, 7.4 MiB/s wr, 783 op/s 2024-09-16T08:14:17.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:14:17 smithi144 ceph-mon[25668]: pgmap v273: 97 pgs: 97 active+clean; 252 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 13 KiB/s rd, 7.4 MiB/s wr, 783 op/s 2024-09-16T08:14:17.996 INFO:teuthology.orchestra.run.smithi028.stdout: 5 43870 16.65 MB/sec warmup 76 sec latency 41.911 ms 2024-09-16T08:14:18.413 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:14:18 smithi028 ceph-mon[23744]: pgmap v274: 97 pgs: 97 active+clean; 258 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 8.2 KiB/s rd, 7.5 MiB/s wr, 641 op/s 2024-09-16T08:14:18.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:14:18 smithi144 ceph-mon[25668]: pgmap v274: 97 pgs: 97 active+clean; 258 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 8.2 KiB/s rd, 7.5 MiB/s wr, 641 op/s 2024-09-16T08:14:18.996 INFO:teuthology.orchestra.run.smithi028.stdout: 5 44181 16.50 MB/sec warmup 77 sec latency 75.620 ms 2024-09-16T08:14:19.996 INFO:teuthology.orchestra.run.smithi028.stdout: 5 44513 16.43 MB/sec warmup 78 sec latency 58.282 ms 2024-09-16T08:14:21.001 INFO:teuthology.orchestra.run.smithi028.stdout: 5 45112 16.39 MB/sec warmup 79 sec latency 67.541 ms 2024-09-16T08:14:21.277 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:14:21 smithi028 ceph-mon[23744]: pgmap v275: 97 pgs: 97 active+clean; 258 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 8.0 KiB/s rd, 5.6 MiB/s wr, 523 op/s 2024-09-16T08:14:21.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:14:21 smithi144 ceph-mon[25668]: pgmap v275: 97 pgs: 97 active+clean; 258 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 8.0 KiB/s rd, 5.6 MiB/s wr, 523 op/s 2024-09-16T08:14:22.002 INFO:teuthology.orchestra.run.smithi028.stdout: 5 45741 16.49 MB/sec warmup 80 sec latency 56.533 ms 2024-09-16T08:14:23.002 INFO:teuthology.orchestra.run.smithi028.stdout: 5 46002 16.40 MB/sec warmup 81 sec latency 63.924 ms 2024-09-16T08:14:23.306 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:14:23 smithi028 ceph-mon[23744]: pgmap v276: 97 pgs: 97 active+clean; 258 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 6.6 MiB/s wr, 656 op/s 2024-09-16T08:14:23.442 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:14:23 smithi144 ceph-mon[25668]: pgmap v276: 97 pgs: 97 active+clean; 258 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 6.6 MiB/s wr, 656 op/s 2024-09-16T08:14:24.008 INFO:teuthology.orchestra.run.smithi028.stdout: 5 46540 16.41 MB/sec warmup 82 sec latency 43.576 ms 2024-09-16T08:14:25.008 INFO:teuthology.orchestra.run.smithi028.stdout: 5 47182 16.37 MB/sec warmup 83 sec latency 22.061 ms 2024-09-16T08:14:25.327 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:14:25 smithi028 ceph-mon[23744]: pgmap v277: 97 pgs: 97 active+clean; 262 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 9.2 KiB/s rd, 6.8 MiB/s wr, 682 op/s 2024-09-16T08:14:25.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:14:25 smithi144 ceph-mon[25668]: pgmap v277: 97 pgs: 97 active+clean; 262 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 9.2 KiB/s rd, 6.8 MiB/s wr, 682 op/s 2024-09-16T08:14:26.008 INFO:teuthology.orchestra.run.smithi028.stdout: 5 47941 16.35 MB/sec warmup 84 sec latency 24.467 ms 2024-09-16T08:14:27.008 INFO:teuthology.orchestra.run.smithi028.stdout: 5 48843 16.41 MB/sec warmup 85 sec latency 26.496 ms 2024-09-16T08:14:27.164 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:14:27 smithi028 ceph-mon[23744]: pgmap v278: 97 pgs: 97 active+clean; 264 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 5.7 MiB/s wr, 526 op/s 2024-09-16T08:14:27.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:14:27 smithi144 ceph-mon[25668]: pgmap v278: 97 pgs: 97 active+clean; 264 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 5.7 MiB/s wr, 526 op/s 2024-09-16T08:14:28.008 INFO:teuthology.orchestra.run.smithi028.stdout: 5 49592 16.58 MB/sec warmup 86 sec latency 30.810 ms 2024-09-16T08:14:29.008 INFO:teuthology.orchestra.run.smithi028.stdout: 5 50238 16.60 MB/sec warmup 87 sec latency 19.316 ms 2024-09-16T08:14:29.355 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:14:29 smithi028 ceph-mon[23744]: pgmap v279: 97 pgs: 97 active+clean; 278 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 6.7 KiB/s rd, 7.4 MiB/s wr, 647 op/s 2024-09-16T08:14:29.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:14:29 smithi144 ceph-mon[25668]: pgmap v279: 97 pgs: 97 active+clean; 278 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 6.7 KiB/s rd, 7.4 MiB/s wr, 647 op/s 2024-09-16T08:14:30.008 INFO:teuthology.orchestra.run.smithi028.stdout: 5 50938 16.55 MB/sec warmup 88 sec latency 12.465 ms 2024-09-16T08:14:31.008 INFO:teuthology.orchestra.run.smithi028.stdout: 5 51712 16.56 MB/sec warmup 89 sec latency 23.635 ms 2024-09-16T08:14:31.368 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:14:31 smithi028 ceph-mon[23744]: pgmap v280: 97 pgs: 97 active+clean; 278 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 6.6 KiB/s rd, 5.8 MiB/s wr, 568 op/s 2024-09-16T08:14:31.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:14:31 smithi144 ceph-mon[25668]: pgmap v280: 97 pgs: 97 active+clean; 278 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 6.6 KiB/s rd, 5.8 MiB/s wr, 568 op/s 2024-09-16T08:14:32.008 INFO:teuthology.orchestra.run.smithi028.stdout: 5 52509 16.60 MB/sec warmup 90 sec latency 31.185 ms 2024-09-16T08:14:33.008 INFO:teuthology.orchestra.run.smithi028.stdout: 5 53238 16.74 MB/sec warmup 91 sec latency 22.143 ms 2024-09-16T08:14:33.381 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:14:33 smithi028 ceph-mon[23744]: pgmap v281: 97 pgs: 97 active+clean; 286 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 8.0 MiB/s wr, 804 op/s 2024-09-16T08:14:33.442 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:14:33 smithi144 ceph-mon[25668]: pgmap v281: 97 pgs: 97 active+clean; 286 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 8.0 MiB/s wr, 804 op/s 2024-09-16T08:14:33.831 INFO:teuthology.orchestra.run.smithi028.stdout:true 2024-09-16T08:14:34.008 INFO:teuthology.orchestra.run.smithi028.stdout: 5 53789 16.73 MB/sec warmup 92 sec latency 32.625 ms 2024-09-16T08:14:34.884 INFO:teuthology.orchestra.run.smithi028.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-16T08:14:34.884 INFO:teuthology.orchestra.run.smithi028.stdout:alertmanager.smithi028 smithi028 *:9093,9094 running (6m) 95s ago 7m 24.7M - ba2b418f427c 8e1f82e9293c 2024-09-16T08:14:34.884 INFO:teuthology.orchestra.run.smithi028.stdout:crash.smithi028 smithi028 running (7m) 95s ago 7m 6405k - 17.2.0 e1d6a67b021e 69e795c000fd 2024-09-16T08:14:34.884 INFO:teuthology.orchestra.run.smithi028.stdout:crash.smithi144 smithi144 running (7m) 97s ago 7m 6400k - 17.2.0 e1d6a67b021e bbe54b51649c 2024-09-16T08:14:34.884 INFO:teuthology.orchestra.run.smithi028.stdout:grafana.smithi028 smithi028 *:3000 running (6m) 95s ago 7m 52.7M - 8.3.5 dad864ee21e9 6ff027b7efc7 2024-09-16T08:14:34.885 INFO:teuthology.orchestra.run.smithi028.stdout:mds.foofs.smithi028.srtgev smithi028 running (2m) 95s ago 2m 14.9M - 17.2.0 e1d6a67b021e 250e7a7f754c 2024-09-16T08:14:34.885 INFO:teuthology.orchestra.run.smithi028.stdout:mds.foofs.smithi144.oozgsi smithi144 running (2m) 97s ago 2m 14.4M - 17.2.0 e1d6a67b021e ecf4c2bb0fdc 2024-09-16T08:14:34.885 INFO:teuthology.orchestra.run.smithi028.stdout:mgr.smithi028.azvyzy smithi028 *:9283 running (9m) 95s ago 9m 460M - 17.2.0 e1d6a67b021e 229e5e86f252 2024-09-16T08:14:34.885 INFO:teuthology.orchestra.run.smithi028.stdout:mgr.smithi144.utzszj smithi144 *:8443,9283 running (7m) 97s ago 7m 412M - 17.2.0 e1d6a67b021e ba5c5c8ef9ca 2024-09-16T08:14:34.885 INFO:teuthology.orchestra.run.smithi028.stdout:mon.smithi028 smithi028 running (9m) 95s ago 9m 46.3M 2048M 17.2.0 e1d6a67b021e a2f688697052 2024-09-16T08:14:34.885 INFO:teuthology.orchestra.run.smithi028.stdout:mon.smithi144 smithi144 running (7m) 97s ago 7m 36.8M 2048M 17.2.0 e1d6a67b021e 1c80e3ff3507 2024-09-16T08:14:34.885 INFO:teuthology.orchestra.run.smithi028.stdout:nfs.foo.0.0.smithi028.dkttgj smithi028 *:2049 running (105s) 95s ago 105s 49.0M - 4.0 e1d6a67b021e b5dccc45a6ed 2024-09-16T08:14:34.885 INFO:teuthology.orchestra.run.smithi028.stdout:nfs.foo.1.0.smithi144.epipkg smithi144 *:2049 running (100s) 97s ago 100s 50.7M - 4.0 e1d6a67b021e faa89e61f50e 2024-09-16T08:14:34.885 INFO:teuthology.orchestra.run.smithi028.stdout:node-exporter.smithi028 smithi028 *:9100 running (7m) 95s ago 7m 17.0M - 1dbe0e931976 7ad46df29362 2024-09-16T08:14:34.885 INFO:teuthology.orchestra.run.smithi028.stdout:node-exporter.smithi144 smithi144 *:9100 running (7m) 97s ago 7m 18.6M - 1dbe0e931976 f85c6daf2fe0 2024-09-16T08:14:34.885 INFO:teuthology.orchestra.run.smithi028.stdout:osd.0 smithi028 running (6m) 95s ago 6m 43.2M 3217M 17.2.0 e1d6a67b021e 8184fbc3871f 2024-09-16T08:14:34.885 INFO:teuthology.orchestra.run.smithi028.stdout:osd.1 smithi028 running (6m) 95s ago 6m 44.3M 3217M 17.2.0 e1d6a67b021e 6911d5c68d54 2024-09-16T08:14:34.885 INFO:teuthology.orchestra.run.smithi028.stdout:osd.2 smithi028 running (5m) 95s ago 5m 39.0M 3217M 17.2.0 e1d6a67b021e 90b27d8375a2 2024-09-16T08:14:34.886 INFO:teuthology.orchestra.run.smithi028.stdout:osd.3 smithi028 running (5m) 95s ago 5m 42.6M 3217M 17.2.0 e1d6a67b021e dac477d8b04f 2024-09-16T08:14:34.886 INFO:teuthology.orchestra.run.smithi028.stdout:osd.4 smithi144 running (4m) 97s ago 4m 40.0M 3984M 17.2.0 e1d6a67b021e 86719c60975c 2024-09-16T08:14:34.886 INFO:teuthology.orchestra.run.smithi028.stdout:osd.5 smithi144 running (4m) 97s ago 4m 38.7M 3984M 17.2.0 e1d6a67b021e 70452c145c60 2024-09-16T08:14:34.886 INFO:teuthology.orchestra.run.smithi028.stdout:osd.6 smithi144 running (4m) 97s ago 4m 40.2M 3984M 17.2.0 e1d6a67b021e 7f9981be792f 2024-09-16T08:14:34.886 INFO:teuthology.orchestra.run.smithi028.stdout:osd.7 smithi144 running (3m) 97s ago 3m 41.8M 3984M 17.2.0 e1d6a67b021e dada74ddbe4c 2024-09-16T08:14:34.886 INFO:teuthology.orchestra.run.smithi028.stdout:prometheus.smithi028 smithi028 *:9095 running (6m) 95s ago 6m 63.2M - 514e6a882f6e fd3df637374d 2024-09-16T08:14:35.008 INFO:teuthology.orchestra.run.smithi028.stdout: 5 54321 16.67 MB/sec warmup 93 sec latency 37.652 ms 2024-09-16T08:14:35.153 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:14:35 smithi028 ceph-mon[23744]: pgmap v282: 97 pgs: 97 active+clean; 289 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 7.9 KiB/s rd, 9.3 MiB/s wr, 843 op/s 2024-09-16T08:14:35.153 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:14:35 smithi028 ceph-mon[23744]: from='client.14660 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T08:14:35.581 INFO:teuthology.orchestra.run.smithi028.stdout:{ 2024-09-16T08:14:35.581 INFO:teuthology.orchestra.run.smithi028.stdout: "mon": { 2024-09-16T08:14:35.581 INFO:teuthology.orchestra.run.smithi028.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-16T08:14:35.581 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-09-16T08:14:35.581 INFO:teuthology.orchestra.run.smithi028.stdout: "mgr": { 2024-09-16T08:14:35.581 INFO:teuthology.orchestra.run.smithi028.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-16T08:14:35.581 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-09-16T08:14:35.581 INFO:teuthology.orchestra.run.smithi028.stdout: "osd": { 2024-09-16T08:14:35.581 INFO:teuthology.orchestra.run.smithi028.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-09-16T08:14:35.581 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-09-16T08:14:35.581 INFO:teuthology.orchestra.run.smithi028.stdout: "mds": { 2024-09-16T08:14:35.581 INFO:teuthology.orchestra.run.smithi028.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-16T08:14:35.582 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-09-16T08:14:35.582 INFO:teuthology.orchestra.run.smithi028.stdout: "overall": { 2024-09-16T08:14:35.582 INFO:teuthology.orchestra.run.smithi028.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 14 2024-09-16T08:14:35.582 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-09-16T08:14:35.582 INFO:teuthology.orchestra.run.smithi028.stdout:} 2024-09-16T08:14:35.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:14:35 smithi144 ceph-mon[25668]: pgmap v282: 97 pgs: 97 active+clean; 289 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 7.9 KiB/s rd, 9.3 MiB/s wr, 843 op/s 2024-09-16T08:14:35.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:14:35 smithi144 ceph-mon[25668]: from='client.14660 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T08:14:36.009 INFO:teuthology.orchestra.run.smithi028.stdout: 5 54922 16.60 MB/sec warmup 94 sec latency 34.611 ms 2024-09-16T08:14:36.120 INFO:teuthology.orchestra.run.smithi028.stdout:{ 2024-09-16T08:14:36.120 INFO:teuthology.orchestra.run.smithi028.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:f7ad55f9ccd17ba66aadb19b4d291e67b60ea24e5d8a0c0e92941fd59c81e4ce", 2024-09-16T08:14:36.120 INFO:teuthology.orchestra.run.smithi028.stdout: "in_progress": true, 2024-09-16T08:14:36.120 INFO:teuthology.orchestra.run.smithi028.stdout: "services_complete": [], 2024-09-16T08:14:36.121 INFO:teuthology.orchestra.run.smithi028.stdout: "progress": "0/23 daemons upgraded", 2024-09-16T08:14:36.121 INFO:teuthology.orchestra.run.smithi028.stdout: "message": "Pulling quay.ceph.io/ceph-ci/ceph@sha256:f7ad55f9ccd17ba66aadb19b4d291e67b60ea24e5d8a0c0e92941fd59c81e4ce image on host smithi144" 2024-09-16T08:14:36.121 INFO:teuthology.orchestra.run.smithi028.stdout:} 2024-09-16T08:14:36.167 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:14:36 smithi028 ceph-mon[23744]: from='client.24439 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T08:14:36.167 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:14:36 smithi028 ceph-mon[23744]: from='client.24443 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T08:14:36.168 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:14:36 smithi028 ceph-mon[23744]: from='client.? 172.21.15.28:0/4267317710' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-16T08:14:36.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:14:36 smithi144 ceph-mon[25668]: from='client.24439 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T08:14:36.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:14:36 smithi144 ceph-mon[25668]: from='client.24443 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T08:14:36.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:14:36 smithi144 ceph-mon[25668]: from='client.? 172.21.15.28:0/4267317710' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-16T08:14:36.736 INFO:teuthology.orchestra.run.smithi028.stdout:HEALTH_OK 2024-09-16T08:14:37.008 INFO:teuthology.orchestra.run.smithi028.stdout: 5 55616 16.61 MB/sec warmup 95 sec latency 43.229 ms 2024-09-16T08:14:37.195 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:14:37 smithi028 ceph-mon[23744]: pgmap v283: 97 pgs: 97 active+clean; 294 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 14 KiB/s rd, 9.0 MiB/s wr, 821 op/s 2024-09-16T08:14:37.233 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:14:37 smithi028 ceph-mon[23744]: from='client.24451 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T08:14:37.233 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:14:37 smithi028 ceph-mon[23744]: from='client.? 172.21.15.28:0/3891715119' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-16T08:14:37.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:14:37 smithi144 ceph-mon[25668]: pgmap v283: 97 pgs: 97 active+clean; 294 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 14 KiB/s rd, 9.0 MiB/s wr, 821 op/s 2024-09-16T08:14:37.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:14:37 smithi144 ceph-mon[25668]: from='client.24451 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T08:14:37.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:14:37 smithi144 ceph-mon[25668]: from='client.? 172.21.15.28:0/3891715119' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-16T08:14:38.008 INFO:teuthology.orchestra.run.smithi028.stdout: 5 56079 16.57 MB/sec warmup 96 sec latency 33.425 ms 2024-09-16T08:14:39.009 INFO:teuthology.orchestra.run.smithi028.stdout: 5 56655 16.67 MB/sec warmup 97 sec latency 52.891 ms 2024-09-16T08:14:39.236 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:14:39 smithi028 ceph-mon[23744]: pgmap v284: 97 pgs: 97 active+clean; 297 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 11 MiB/s wr, 1.03k op/s 2024-09-16T08:14:39.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:14:39 smithi144 ceph-mon[25668]: pgmap v284: 97 pgs: 97 active+clean; 297 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 11 MiB/s wr, 1.03k op/s 2024-09-16T08:14:40.009 INFO:teuthology.orchestra.run.smithi028.stdout: 5 57003 16.63 MB/sec warmup 98 sec latency 140.318 ms 2024-09-16T08:14:41.009 INFO:teuthology.orchestra.run.smithi028.stdout: 5 57236 16.50 MB/sec warmup 99 sec latency 108.712 ms 2024-09-16T08:14:41.270 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:14:41 smithi028 ceph-mon[23744]: pgmap v285: 97 pgs: 97 active+clean; 297 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 8.2 MiB/s wr, 803 op/s 2024-09-16T08:14:41.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:14:41 smithi144 ceph-mon[25668]: pgmap v285: 97 pgs: 97 active+clean; 297 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 8.2 MiB/s wr, 803 op/s 2024-09-16T08:14:42.009 INFO:teuthology.orchestra.run.smithi028.stdout: 5 57576 16.38 MB/sec warmup 100 sec latency 107.523 ms 2024-09-16T08:14:43.009 INFO:teuthology.orchestra.run.smithi028.stdout: 5 58140 16.34 MB/sec warmup 101 sec latency 48.283 ms 2024-09-16T08:14:43.293 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:14:43 smithi028 ceph-mon[23744]: pgmap v286: 97 pgs: 97 active+clean; 302 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 17 KiB/s rd, 9.9 MiB/s wr, 966 op/s 2024-09-16T08:14:43.444 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:14:43 smithi144 ceph-mon[25668]: pgmap v286: 97 pgs: 97 active+clean; 302 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 17 KiB/s rd, 9.9 MiB/s wr, 966 op/s 2024-09-16T08:14:44.009 INFO:teuthology.orchestra.run.smithi028.stdout: 5 58735 16.32 MB/sec warmup 102 sec latency 31.220 ms 2024-09-16T08:14:45.009 INFO:teuthology.orchestra.run.smithi028.stdout: 5 59518 16.34 MB/sec warmup 103 sec latency 24.029 ms 2024-09-16T08:14:45.307 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:14:45 smithi028 ceph-mon[23744]: pgmap v287: 97 pgs: 97 active+clean; 308 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 9.2 MiB/s wr, 846 op/s 2024-09-16T08:14:45.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:14:45 smithi144 ceph-mon[25668]: pgmap v287: 97 pgs: 97 active+clean; 308 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 9.2 MiB/s wr, 846 op/s 2024-09-16T08:14:46.009 INFO:teuthology.orchestra.run.smithi028.stdout: 5 60328 16.50 MB/sec warmup 104 sec latency 24.895 ms 2024-09-16T08:14:47.009 INFO:teuthology.orchestra.run.smithi028.stdout: 5 60974 16.50 MB/sec warmup 105 sec latency 34.769 ms 2024-09-16T08:14:47.330 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:14:47 smithi028 ceph-mon[23744]: pgmap v288: 97 pgs: 97 active+clean; 310 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 16 KiB/s rd, 7.9 MiB/s wr, 797 op/s 2024-09-16T08:14:47.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:14:47 smithi144 ceph-mon[25668]: pgmap v288: 97 pgs: 97 active+clean; 310 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 16 KiB/s rd, 7.9 MiB/s wr, 797 op/s 2024-09-16T08:14:48.009 INFO:teuthology.orchestra.run.smithi028.stdout: 5 61686 16.46 MB/sec warmup 106 sec latency 33.775 ms 2024-09-16T08:14:49.009 INFO:teuthology.orchestra.run.smithi028.stdout: 5 62395 16.46 MB/sec warmup 107 sec latency 28.551 ms 2024-09-16T08:14:49.342 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:14:49 smithi028 ceph-mon[23744]: pgmap v289: 97 pgs: 97 active+clean; 319 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 10 KiB/s rd, 9.5 MiB/s wr, 925 op/s 2024-09-16T08:14:49.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:14:49 smithi144 ceph-mon[25668]: pgmap v289: 97 pgs: 97 active+clean; 319 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 10 KiB/s rd, 9.5 MiB/s wr, 925 op/s 2024-09-16T08:14:50.009 INFO:teuthology.orchestra.run.smithi028.stdout: 5 63127 16.49 MB/sec warmup 108 sec latency 25.437 ms 2024-09-16T08:14:51.009 INFO:teuthology.orchestra.run.smithi028.stdout: 5 63800 16.59 MB/sec warmup 109 sec latency 22.502 ms 2024-09-16T08:14:51.109 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:14:50 smithi028 ceph-mon[23744]: pgmap v290: 97 pgs: 97 active+clean; 319 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 10 KiB/s rd, 7.1 MiB/s wr, 683 op/s 2024-09-16T08:14:51.110 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:14:50 smithi028 ceph-mon[23744]: Upgrade: Updating mgr.smithi144.utzszj 2024-09-16T08:14:51.110 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:14:50 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' 2024-09-16T08:14:51.110 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:14:50 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi144.utzszj", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-16T08:14:51.110 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:14:50 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-16T08:14:51.110 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:14:50 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T08:14:51.110 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:14:50 smithi028 ceph-mon[23744]: Deploying daemon mgr.smithi144.utzszj on smithi144 2024-09-16T08:14:51.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:14:50 smithi144 ceph-mon[25668]: pgmap v290: 97 pgs: 97 active+clean; 319 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 10 KiB/s rd, 7.1 MiB/s wr, 683 op/s 2024-09-16T08:14:51.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:14:50 smithi144 ceph-mon[25668]: Upgrade: Updating mgr.smithi144.utzszj 2024-09-16T08:14:51.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:14:50 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' 2024-09-16T08:14:51.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:14:50 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi144.utzszj", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-16T08:14:51.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:14:50 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-16T08:14:51.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:14:50 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T08:14:51.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:14:50 smithi144 ceph-mon[25668]: Deploying daemon mgr.smithi144.utzszj on smithi144 2024-09-16T08:14:52.009 INFO:teuthology.orchestra.run.smithi028.stdout: 5 64366 16.60 MB/sec warmup 110 sec latency 32.915 ms 2024-09-16T08:14:53.009 INFO:teuthology.orchestra.run.smithi028.stdout: 5 64932 16.55 MB/sec warmup 111 sec latency 28.703 ms 2024-09-16T08:14:53.081 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:14:52 smithi144 ceph-mon[25668]: pgmap v291: 97 pgs: 97 active+clean; 320 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 16 KiB/s rd, 8.8 MiB/s wr, 849 op/s 2024-09-16T08:14:53.122 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:14:52 smithi028 ceph-mon[23744]: pgmap v291: 97 pgs: 97 active+clean; 320 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 16 KiB/s rd, 8.8 MiB/s wr, 849 op/s 2024-09-16T08:14:54.009 INFO:teuthology.orchestra.run.smithi028.stdout: 5 65539 16.49 MB/sec warmup 112 sec latency 24.901 ms 2024-09-16T08:14:54.887 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:14:54 smithi028 ceph-mon[23744]: pgmap v292: 97 pgs: 97 active+clean; 331 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 9.6 MiB/s wr, 901 op/s 2024-09-16T08:14:54.887 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:14:54 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' 2024-09-16T08:14:54.887 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:14:54 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-16T08:14:54.887 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:14:54 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T08:14:54.887 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:14:54 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-16T08:14:55.009 INFO:teuthology.orchestra.run.smithi028.stdout: 5 66196 16.51 MB/sec warmup 113 sec latency 48.680 ms 2024-09-16T08:14:55.081 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:14:54 smithi144 ceph-mon[25668]: pgmap v292: 97 pgs: 97 active+clean; 331 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 9.6 MiB/s wr, 901 op/s 2024-09-16T08:14:55.082 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:14:54 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' 2024-09-16T08:14:55.082 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:14:54 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-16T08:14:55.082 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:14:54 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T08:14:55.082 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:14:54 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-16T08:14:56.009 INFO:teuthology.orchestra.run.smithi028.stdout: 5 66853 16.55 MB/sec warmup 114 sec latency 29.672 ms 2024-09-16T08:14:57.009 INFO:teuthology.orchestra.run.smithi028.stdout: 5 67423 16.57 MB/sec warmup 115 sec latency 70.854 ms 2024-09-16T08:14:57.195 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:14:57 smithi028 ceph-mon[23744]: pgmap v293: 97 pgs: 97 active+clean; 333 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 15 KiB/s rd, 8.8 MiB/s wr, 819 op/s 2024-09-16T08:14:57.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:14:57 smithi144 ceph-mon[25668]: pgmap v293: 97 pgs: 97 active+clean; 333 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 15 KiB/s rd, 8.8 MiB/s wr, 819 op/s 2024-09-16T08:14:58.009 INFO:teuthology.orchestra.run.smithi028.stdout: 5 67834 16.54 MB/sec warmup 116 sec latency 54.577 ms 2024-09-16T08:14:59.009 INFO:teuthology.orchestra.run.smithi028.stdout: 5 68099 16.45 MB/sec warmup 117 sec latency 62.773 ms 2024-09-16T08:14:59.218 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:14:59 smithi028 ceph-mon[23744]: pgmap v294: 97 pgs: 97 active+clean; 339 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 11 MiB/s wr, 1.03k op/s 2024-09-16T08:14:59.219 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:14:59 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' 2024-09-16T08:14:59.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:14:59 smithi144 ceph-mon[25668]: pgmap v294: 97 pgs: 97 active+clean; 339 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 11 MiB/s wr, 1.03k op/s 2024-09-16T08:14:59.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:14:59 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' 2024-09-16T08:15:00.009 INFO:teuthology.orchestra.run.smithi028.stdout: 5 68425 16.35 MB/sec warmup 118 sec latency 61.875 ms 2024-09-16T08:15:00.736 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:15:00 smithi028 ceph-mon[23744]: pgmap v295: 97 pgs: 97 active+clean; 339 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 7.8 MiB/s wr, 753 op/s 2024-09-16T08:15:00.737 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:15:00 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' 2024-09-16T08:15:00.737 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:15:00 smithi028 ceph-mon[23744]: from='mgr.? 172.21.15.144:0/4176577021' entity='mgr.smithi144.utzszj' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi144.utzszj/crt"}]: dispatch 2024-09-16T08:15:00.737 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:15:00 smithi028 ceph-mon[23744]: from='mgr.? 172.21.15.144:0/4176577021' entity='mgr.smithi144.utzszj' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-16T08:15:00.737 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:15:00 smithi028 ceph-mon[23744]: from='mgr.? 172.21.15.144:0/4176577021' entity='mgr.smithi144.utzszj' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi144.utzszj/key"}]: dispatch 2024-09-16T08:15:00.737 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:15:00 smithi028 ceph-mon[23744]: from='mgr.? 172.21.15.144:0/4176577021' entity='mgr.smithi144.utzszj' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-16T08:15:00.737 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:15:00 smithi028 ceph-mon[23744]: Standby manager daemon smithi144.utzszj restarted 2024-09-16T08:15:00.737 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:15:00 smithi028 ceph-mon[23744]: Standby manager daemon smithi144.utzszj started 2024-09-16T08:15:01.009 INFO:teuthology.orchestra.run.smithi028.stdout: 5 68796 16.27 MB/sec warmup 119 sec latency 52.374 ms 2024-09-16T08:15:01.081 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:15:00 smithi144 ceph-mon[25668]: pgmap v295: 97 pgs: 97 active+clean; 339 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 7.8 MiB/s wr, 753 op/s 2024-09-16T08:15:01.082 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:15:00 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' 2024-09-16T08:15:01.082 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:15:00 smithi144 ceph-mon[25668]: from='mgr.? 172.21.15.144:0/4176577021' entity='mgr.smithi144.utzszj' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi144.utzszj/crt"}]: dispatch 2024-09-16T08:15:01.082 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:15:00 smithi144 ceph-mon[25668]: from='mgr.? 172.21.15.144:0/4176577021' entity='mgr.smithi144.utzszj' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-16T08:15:01.082 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:15:00 smithi144 ceph-mon[25668]: from='mgr.? 172.21.15.144:0/4176577021' entity='mgr.smithi144.utzszj' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi144.utzszj/key"}]: dispatch 2024-09-16T08:15:01.082 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:15:00 smithi144 ceph-mon[25668]: from='mgr.? 172.21.15.144:0/4176577021' entity='mgr.smithi144.utzszj' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-16T08:15:01.082 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:15:00 smithi144 ceph-mon[25668]: Standby manager daemon smithi144.utzszj restarted 2024-09-16T08:15:01.082 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:15:00 smithi144 ceph-mon[25668]: Standby manager daemon smithi144.utzszj started 2024-09-16T08:15:02.310 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:15:02 smithi144 ceph-mon[25668]: mgrmap e22: smithi028.azvyzy(active, since 7m), standbys: smithi144.utzszj 2024-09-16T08:15:02.321 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:15:02 smithi028 ceph-mon[23744]: mgrmap e22: smithi028.azvyzy(active, since 7m), standbys: smithi144.utzszj 2024-09-16T08:15:03.009 INFO:teuthology.orchestra.run.smithi028.stdout: 5 69569 12.75 MB/sec execute 1 sec latency 64.362 ms 2024-09-16T08:15:03.306 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:15:03 smithi144 ceph-mon[25668]: pgmap v296: 97 pgs: 97 active+clean; 344 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 15 KiB/s rd, 9.1 MiB/s wr, 920 op/s 2024-09-16T08:15:03.335 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:15:03 smithi028 ceph-mon[23744]: pgmap v296: 97 pgs: 97 active+clean; 344 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 15 KiB/s rd, 9.1 MiB/s wr, 920 op/s 2024-09-16T08:15:04.010 INFO:teuthology.orchestra.run.smithi028.stdout: 5 70119 12.81 MB/sec execute 2 sec latency 71.154 ms 2024-09-16T08:15:04.392 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:15:04 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi028.azvyzy/mirror_snapshot_schedule"}]: dispatch 2024-09-16T08:15:04.392 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:15:04 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi028.azvyzy/trash_purge_schedule"}]: dispatch 2024-09-16T08:15:04.392 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:15:04 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' 2024-09-16T08:15:04.392 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:15:04 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' 2024-09-16T08:15:04.392 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:15:04 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' 2024-09-16T08:15:04.392 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:15:04 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-16T08:15:04.392 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:15:04 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "mgr fail", "who": "smithi028.azvyzy"}]: dispatch 2024-09-16T08:15:04.392 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:15:04 smithi028 ceph-mon[23744]: osdmap e70: 8 total, 8 up, 8 in 2024-09-16T08:15:04.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:15:04 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi028.azvyzy/mirror_snapshot_schedule"}]: dispatch 2024-09-16T08:15:04.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:15:04 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi028.azvyzy/trash_purge_schedule"}]: dispatch 2024-09-16T08:15:04.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:15:04 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' 2024-09-16T08:15:04.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:15:04 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' 2024-09-16T08:15:04.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:15:04 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' 2024-09-16T08:15:04.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:15:04 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-16T08:15:04.583 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:15:04 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "mgr fail", "who": "smithi028.azvyzy"}]: dispatch 2024-09-16T08:15:04.583 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:15:04 smithi144 ceph-mon[25668]: osdmap e70: 8 total, 8 up, 8 in 2024-09-16T08:15:05.010 INFO:teuthology.orchestra.run.smithi028.stdout: 5 70638 16.55 MB/sec execute 3 sec latency 52.338 ms 2024-09-16T08:15:05.408 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:15:05 smithi028 ceph-mon[23744]: pgmap v297: 97 pgs: 97 active+clean; 341 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 9.1 KiB/s rd, 8.0 MiB/s wr, 795 op/s 2024-09-16T08:15:05.408 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:15:05 smithi028 ceph-mon[23744]: Upgrade: Need to upgrade myself (mgr.smithi028.azvyzy) 2024-09-16T08:15:05.408 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:15:05 smithi028 ceph-mon[23744]: Failing over to other MGR 2024-09-16T08:15:05.408 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:15:05 smithi028 ceph-mon[23744]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd='[{"prefix": "mgr fail", "who": "smithi028.azvyzy"}]': finished 2024-09-16T08:15:05.408 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:15:05 smithi028 ceph-mon[23744]: from='mgr.24459 172.21.15.144:0/4176577021' entity='mgr.smithi144.utzszj' cmd=[{"prefix": "mon metadata", "id": "smithi028"}]: dispatch 2024-09-16T08:15:05.408 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:15:05 smithi028 ceph-mon[23744]: mgrmap e23: smithi144.utzszj(active, starting, since 0.998889s) 2024-09-16T08:15:05.408 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:15:05 smithi028 ceph-mon[23744]: from='mgr.24459 172.21.15.144:0/4176577021' entity='mgr.smithi144.utzszj' cmd=[{"prefix": "mon metadata", "id": "smithi144"}]: dispatch 2024-09-16T08:15:05.408 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:15:05 smithi028 ceph-mon[23744]: from='mgr.24459 172.21.15.144:0/4176577021' entity='mgr.smithi144.utzszj' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi028.srtgev"}]: dispatch 2024-09-16T08:15:05.409 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:15:05 smithi028 ceph-mon[23744]: from='mgr.24459 172.21.15.144:0/4176577021' entity='mgr.smithi144.utzszj' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi144.oozgsi"}]: dispatch 2024-09-16T08:15:05.409 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:15:05 smithi028 ceph-mon[23744]: from='mgr.24459 172.21.15.144:0/4176577021' entity='mgr.smithi144.utzszj' cmd=[{"prefix": "mgr metadata", "who": "smithi144.utzszj", "id": "smithi144.utzszj"}]: dispatch 2024-09-16T08:15:05.409 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:15:05 smithi028 ceph-mon[23744]: from='mgr.24459 172.21.15.144:0/4176577021' entity='mgr.smithi144.utzszj' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-16T08:15:05.409 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:15:05 smithi028 ceph-mon[23744]: from='mgr.24459 172.21.15.144:0/4176577021' entity='mgr.smithi144.utzszj' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-16T08:15:05.409 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:15:05 smithi028 ceph-mon[23744]: from='mgr.24459 172.21.15.144:0/4176577021' entity='mgr.smithi144.utzszj' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-16T08:15:05.409 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:15:05 smithi028 ceph-mon[23744]: from='mgr.24459 172.21.15.144:0/4176577021' entity='mgr.smithi144.utzszj' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-16T08:15:05.409 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:15:05 smithi028 ceph-mon[23744]: from='mgr.24459 172.21.15.144:0/4176577021' entity='mgr.smithi144.utzszj' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-16T08:15:05.409 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:15:05 smithi028 ceph-mon[23744]: from='mgr.24459 172.21.15.144:0/4176577021' entity='mgr.smithi144.utzszj' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-16T08:15:05.409 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:15:05 smithi028 ceph-mon[23744]: from='mgr.24459 172.21.15.144:0/4176577021' entity='mgr.smithi144.utzszj' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-16T08:15:05.409 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:15:05 smithi028 ceph-mon[23744]: from='mgr.24459 172.21.15.144:0/4176577021' entity='mgr.smithi144.utzszj' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-16T08:15:05.409 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:15:05 smithi028 ceph-mon[23744]: from='mgr.24459 172.21.15.144:0/4176577021' entity='mgr.smithi144.utzszj' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-16T08:15:05.409 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:15:05 smithi028 ceph-mon[23744]: from='mgr.24459 172.21.15.144:0/4176577021' entity='mgr.smithi144.utzszj' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-16T08:15:05.409 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:15:05 smithi028 ceph-mon[23744]: from='mgr.24459 172.21.15.144:0/4176577021' entity='mgr.smithi144.utzszj' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-16T08:15:05.410 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:15:05 smithi028 ceph-mon[23744]: Manager daemon smithi144.utzszj is now available 2024-09-16T08:15:05.466 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:15:05 smithi144 ceph-mon[25668]: pgmap v297: 97 pgs: 97 active+clean; 341 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 9.1 KiB/s rd, 8.0 MiB/s wr, 795 op/s 2024-09-16T08:15:05.466 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:15:05 smithi144 ceph-mon[25668]: Upgrade: Need to upgrade myself (mgr.smithi028.azvyzy) 2024-09-16T08:15:05.466 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:15:05 smithi144 ceph-mon[25668]: Failing over to other MGR 2024-09-16T08:15:05.466 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:15:05 smithi144 ceph-mon[25668]: from='mgr.14202 172.21.15.28:0/493227000' entity='mgr.smithi028.azvyzy' cmd='[{"prefix": "mgr fail", "who": "smithi028.azvyzy"}]': finished 2024-09-16T08:15:05.466 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:15:05 smithi144 ceph-mon[25668]: from='mgr.24459 172.21.15.144:0/4176577021' entity='mgr.smithi144.utzszj' cmd=[{"prefix": "mon metadata", "id": "smithi028"}]: dispatch 2024-09-16T08:15:05.466 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:15:05 smithi144 ceph-mon[25668]: mgrmap e23: smithi144.utzszj(active, starting, since 0.998889s) 2024-09-16T08:15:05.466 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:15:05 smithi144 ceph-mon[25668]: from='mgr.24459 172.21.15.144:0/4176577021' entity='mgr.smithi144.utzszj' cmd=[{"prefix": "mon metadata", "id": "smithi144"}]: dispatch 2024-09-16T08:15:05.466 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:15:05 smithi144 ceph-mon[25668]: from='mgr.24459 172.21.15.144:0/4176577021' entity='mgr.smithi144.utzszj' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi028.srtgev"}]: dispatch 2024-09-16T08:15:05.466 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:15:05 smithi144 ceph-mon[25668]: from='mgr.24459 172.21.15.144:0/4176577021' entity='mgr.smithi144.utzszj' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi144.oozgsi"}]: dispatch 2024-09-16T08:15:05.466 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:15:05 smithi144 ceph-mon[25668]: from='mgr.24459 172.21.15.144:0/4176577021' entity='mgr.smithi144.utzszj' cmd=[{"prefix": "mgr metadata", "who": "smithi144.utzszj", "id": "smithi144.utzszj"}]: dispatch 2024-09-16T08:15:05.467 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:15:05 smithi144 ceph-mon[25668]: from='mgr.24459 172.21.15.144:0/4176577021' entity='mgr.smithi144.utzszj' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-16T08:15:05.467 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:15:05 smithi144 ceph-mon[25668]: from='mgr.24459 172.21.15.144:0/4176577021' entity='mgr.smithi144.utzszj' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-16T08:15:05.467 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:15:05 smithi144 ceph-mon[25668]: from='mgr.24459 172.21.15.144:0/4176577021' entity='mgr.smithi144.utzszj' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-16T08:15:05.467 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:15:05 smithi144 ceph-mon[25668]: from='mgr.24459 172.21.15.144:0/4176577021' entity='mgr.smithi144.utzszj' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-16T08:15:05.467 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:15:05 smithi144 ceph-mon[25668]: from='mgr.24459 172.21.15.144:0/4176577021' entity='mgr.smithi144.utzszj' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-16T08:15:05.467 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:15:05 smithi144 ceph-mon[25668]: from='mgr.24459 172.21.15.144:0/4176577021' entity='mgr.smithi144.utzszj' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-16T08:15:05.467 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:15:05 smithi144 ceph-mon[25668]: from='mgr.24459 172.21.15.144:0/4176577021' entity='mgr.smithi144.utzszj' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-16T08:15:05.467 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:15:05 smithi144 ceph-mon[25668]: from='mgr.24459 172.21.15.144:0/4176577021' entity='mgr.smithi144.utzszj' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-16T08:15:05.467 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:15:05 smithi144 ceph-mon[25668]: from='mgr.24459 172.21.15.144:0/4176577021' entity='mgr.smithi144.utzszj' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-16T08:15:05.467 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:15:05 smithi144 ceph-mon[25668]: from='mgr.24459 172.21.15.144:0/4176577021' entity='mgr.smithi144.utzszj' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-16T08:15:05.467 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:15:05 smithi144 ceph-mon[25668]: from='mgr.24459 172.21.15.144:0/4176577021' entity='mgr.smithi144.utzszj' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-16T08:15:05.467 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:15:05 smithi144 ceph-mon[25668]: Manager daemon smithi144.utzszj is now available 2024-09-16T08:15:06.009 INFO:teuthology.orchestra.run.smithi028.stdout: 5 71017 15.41 MB/sec execute 4 sec latency 31.984 ms 2024-09-16T08:15:06.415 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:15:06 smithi028 ceph-mon[23744]: from='mgr.24459 ' entity='mgr.smithi144.utzszj' 2024-09-16T08:15:06.415 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:15:06 smithi028 ceph-mon[23744]: from='mgr.24459 ' entity='mgr.smithi144.utzszj' 2024-09-16T08:15:06.415 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:15:06 smithi028 ceph-mon[23744]: from='mgr.24459 ' entity='mgr.smithi144.utzszj' 2024-09-16T08:15:06.415 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:15:06 smithi028 ceph-mon[23744]: Checking for cert/key for grafana.smithi028 2024-09-16T08:15:06.415 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:15:06 smithi028 ceph-mon[23744]: from='mgr.24459 ' entity='mgr.smithi144.utzszj' 2024-09-16T08:15:06.415 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:15:06 smithi028 ceph-mon[23744]: from='mgr.24459 ' entity='mgr.smithi144.utzszj' 2024-09-16T08:15:06.415 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:15:06 smithi028 ceph-mon[23744]: from='mgr.24459 172.21.15.144:0/4176577021' entity='mgr.smithi144.utzszj' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-16T08:15:06.415 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:15:06 smithi028 ceph-mon[23744]: from='mgr.24459 172.21.15.144:0/4176577021' entity='mgr.smithi144.utzszj' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-16T08:15:06.415 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:15:06 smithi028 ceph-mon[23744]: from='mgr.24459 172.21.15.144:0/4176577021' entity='mgr.smithi144.utzszj' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi144.utzszj/mirror_snapshot_schedule"}]: dispatch 2024-09-16T08:15:06.415 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:15:06 smithi028 ceph-mon[23744]: from='mgr.24459 ' entity='mgr.smithi144.utzszj' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi144.utzszj/mirror_snapshot_schedule"}]: dispatch 2024-09-16T08:15:06.415 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:15:06 smithi028 ceph-mon[23744]: from='mgr.24459 172.21.15.144:0/4176577021' entity='mgr.smithi144.utzszj' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi144.utzszj/trash_purge_schedule"}]: dispatch 2024-09-16T08:15:06.416 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:15:06 smithi028 ceph-mon[23744]: from='mgr.24459 ' entity='mgr.smithi144.utzszj' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi144.utzszj/trash_purge_schedule"}]: dispatch 2024-09-16T08:15:06.416 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:15:06 smithi028 ceph-mon[23744]: mgrmap e24: smithi144.utzszj(active, since 2s) 2024-09-16T08:15:06.831 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:15:06 smithi144 ceph-mon[25668]: from='mgr.24459 ' entity='mgr.smithi144.utzszj' 2024-09-16T08:15:06.831 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:15:06 smithi144 ceph-mon[25668]: from='mgr.24459 ' entity='mgr.smithi144.utzszj' 2024-09-16T08:15:06.832 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:15:06 smithi144 ceph-mon[25668]: from='mgr.24459 ' entity='mgr.smithi144.utzszj' 2024-09-16T08:15:06.832 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:15:06 smithi144 ceph-mon[25668]: Checking for cert/key for grafana.smithi028 2024-09-16T08:15:06.832 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:15:06 smithi144 ceph-mon[25668]: from='mgr.24459 ' entity='mgr.smithi144.utzszj' 2024-09-16T08:15:06.832 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:15:06 smithi144 ceph-mon[25668]: from='mgr.24459 ' entity='mgr.smithi144.utzszj' 2024-09-16T08:15:06.832 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:15:06 smithi144 ceph-mon[25668]: from='mgr.24459 172.21.15.144:0/4176577021' entity='mgr.smithi144.utzszj' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-16T08:15:06.832 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:15:06 smithi144 ceph-mon[25668]: from='mgr.24459 172.21.15.144:0/4176577021' entity='mgr.smithi144.utzszj' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-16T08:15:06.832 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:15:06 smithi144 ceph-mon[25668]: from='mgr.24459 172.21.15.144:0/4176577021' entity='mgr.smithi144.utzszj' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi144.utzszj/mirror_snapshot_schedule"}]: dispatch 2024-09-16T08:15:06.832 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:15:06 smithi144 ceph-mon[25668]: from='mgr.24459 ' entity='mgr.smithi144.utzszj' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi144.utzszj/mirror_snapshot_schedule"}]: dispatch 2024-09-16T08:15:06.832 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:15:06 smithi144 ceph-mon[25668]: from='mgr.24459 172.21.15.144:0/4176577021' entity='mgr.smithi144.utzszj' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi144.utzszj/trash_purge_schedule"}]: dispatch 2024-09-16T08:15:06.832 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:15:06 smithi144 ceph-mon[25668]: from='mgr.24459 ' entity='mgr.smithi144.utzszj' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi144.utzszj/trash_purge_schedule"}]: dispatch 2024-09-16T08:15:06.832 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:15:06 smithi144 ceph-mon[25668]: mgrmap e24: smithi144.utzszj(active, since 2s) 2024-09-16T08:15:07.009 INFO:teuthology.orchestra.run.smithi028.stdout: 5 71417 14.59 MB/sec execute 5 sec latency 54.412 ms 2024-09-16T08:15:07.312 INFO:teuthology.orchestra.run.smithi028.stdout:true 2024-09-16T08:15:07.424 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:15:07 smithi028 ceph-mon[23744]: pgmap v3: 97 pgs: 97 active+clean; 344 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2024-09-16T08:15:07.425 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:15:07 smithi028 ceph-mon[23744]: Deploying cephadm binary to smithi144 2024-09-16T08:15:07.425 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:15:07 smithi028 ceph-mon[23744]: mgrmap e25: smithi144.utzszj(active, since 3s) 2024-09-16T08:15:07.651 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:15:07 smithi144 ceph-mon[25668]: pgmap v3: 97 pgs: 97 active+clean; 344 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2024-09-16T08:15:07.651 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:15:07 smithi144 ceph-mon[25668]: Deploying cephadm binary to smithi144 2024-09-16T08:15:07.651 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:15:07 smithi144 ceph-mon[25668]: mgrmap e25: smithi144.utzszj(active, since 3s) 2024-09-16T08:15:08.009 INFO:teuthology.orchestra.run.smithi028.stdout: 5 71712 12.74 MB/sec execute 6 sec latency 43.706 ms 2024-09-16T08:15:08.430 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:15:08 smithi028 ceph-mon[23744]: pgmap v4: 97 pgs: 97 active+clean; 344 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2024-09-16T08:15:08.430 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:15:08 smithi028 ceph-mon[23744]: Deploying cephadm binary to smithi028 2024-09-16T08:15:08.430 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:15:08 smithi028 ceph-mon[23744]: from='client.14700 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T08:15:08.430 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:15:08 smithi028 ceph-mon[23744]: [16/Sep/2024:08:15:07] ENGINE Bus STARTING 2024-09-16T08:15:08.430 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:15:08 smithi028 ceph-mon[23744]: [16/Sep/2024:08:15:07] ENGINE Serving on http://172.21.15.144:8765 2024-09-16T08:15:08.473 INFO:teuthology.orchestra.run.smithi028.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-16T08:15:08.473 INFO:teuthology.orchestra.run.smithi028.stdout:alertmanager.smithi028 smithi028 *:9093,9094 running (7m) 2m ago 8m 24.7M - ba2b418f427c 8e1f82e9293c 2024-09-16T08:15:08.473 INFO:teuthology.orchestra.run.smithi028.stdout:crash.smithi028 smithi028 running (8m) 2m ago 8m 6405k - 17.2.0 e1d6a67b021e 69e795c000fd 2024-09-16T08:15:08.473 INFO:teuthology.orchestra.run.smithi028.stdout:crash.smithi144 smithi144 running (7m) 9s ago 7m 6400k - 17.2.0 e1d6a67b021e bbe54b51649c 2024-09-16T08:15:08.474 INFO:teuthology.orchestra.run.smithi028.stdout:grafana.smithi028 smithi028 *:3000 running (6m) 2m ago 8m 52.7M - 8.3.5 dad864ee21e9 6ff027b7efc7 2024-09-16T08:15:08.474 INFO:teuthology.orchestra.run.smithi028.stdout:mds.foofs.smithi028.srtgev smithi028 running (2m) 2m ago 2m 14.9M - 17.2.0 e1d6a67b021e 250e7a7f754c 2024-09-16T08:15:08.474 INFO:teuthology.orchestra.run.smithi028.stdout:mds.foofs.smithi144.oozgsi smithi144 running (2m) 9s ago 2m 14.8M - 17.2.0 e1d6a67b021e ecf4c2bb0fdc 2024-09-16T08:15:08.474 INFO:teuthology.orchestra.run.smithi028.stdout:mgr.smithi028.azvyzy smithi028 *:9283 running (9m) 2m ago 9m 460M - 17.2.0 e1d6a67b021e 229e5e86f252 2024-09-16T08:15:08.474 INFO:teuthology.orchestra.run.smithi028.stdout:mgr.smithi144.utzszj smithi144 *:8443,9283 running (15s) 9s ago 7m 66.2M - 19.3.0-4917-g8293d73f 425e316ba8eb eecb46cf2442 2024-09-16T08:15:08.474 INFO:teuthology.orchestra.run.smithi028.stdout:mon.smithi028 smithi028 running (9m) 2m ago 9m 46.3M 2048M 17.2.0 e1d6a67b021e a2f688697052 2024-09-16T08:15:08.474 INFO:teuthology.orchestra.run.smithi028.stdout:mon.smithi144 smithi144 running (7m) 9s ago 7m 39.5M 2048M 17.2.0 e1d6a67b021e 1c80e3ff3507 2024-09-16T08:15:08.474 INFO:teuthology.orchestra.run.smithi028.stdout:nfs.foo.0.0.smithi028.dkttgj smithi028 *:2049 running (2m) 2m ago 2m 49.0M - 4.0 e1d6a67b021e b5dccc45a6ed 2024-09-16T08:15:08.474 INFO:teuthology.orchestra.run.smithi028.stdout:nfs.foo.1.0.smithi144.epipkg smithi144 *:2049 running (2m) 9s ago 2m 75.8M - 4.0 e1d6a67b021e faa89e61f50e 2024-09-16T08:15:08.474 INFO:teuthology.orchestra.run.smithi028.stdout:node-exporter.smithi028 smithi028 *:9100 running (8m) 2m ago 8m 17.0M - 1dbe0e931976 7ad46df29362 2024-09-16T08:15:08.474 INFO:teuthology.orchestra.run.smithi028.stdout:node-exporter.smithi144 smithi144 *:9100 running (7m) 9s ago 7m 18.3M - 1dbe0e931976 f85c6daf2fe0 2024-09-16T08:15:08.474 INFO:teuthology.orchestra.run.smithi028.stdout:osd.0 smithi028 running (7m) 2m ago 7m 43.2M 3217M 17.2.0 e1d6a67b021e 8184fbc3871f 2024-09-16T08:15:08.474 INFO:teuthology.orchestra.run.smithi028.stdout:osd.1 smithi028 running (6m) 2m ago 6m 44.3M 3217M 17.2.0 e1d6a67b021e 6911d5c68d54 2024-09-16T08:15:08.474 INFO:teuthology.orchestra.run.smithi028.stdout:osd.2 smithi028 running (6m) 2m ago 6m 39.0M 3217M 17.2.0 e1d6a67b021e 90b27d8375a2 2024-09-16T08:15:08.475 INFO:teuthology.orchestra.run.smithi028.stdout:osd.3 smithi028 running (5m) 2m ago 5m 42.6M 3217M 17.2.0 e1d6a67b021e dac477d8b04f 2024-09-16T08:15:08.475 INFO:teuthology.orchestra.run.smithi028.stdout:osd.4 smithi144 running (5m) 9s ago 5m 238M 3984M 17.2.0 e1d6a67b021e 86719c60975c 2024-09-16T08:15:08.475 INFO:teuthology.orchestra.run.smithi028.stdout:osd.5 smithi144 running (4m) 9s ago 4m 276M 3984M 17.2.0 e1d6a67b021e 70452c145c60 2024-09-16T08:15:08.475 INFO:teuthology.orchestra.run.smithi028.stdout:osd.6 smithi144 running (4m) 9s ago 4m 247M 3984M 17.2.0 e1d6a67b021e 7f9981be792f 2024-09-16T08:15:08.475 INFO:teuthology.orchestra.run.smithi028.stdout:osd.7 smithi144 running (4m) 9s ago 4m 350M 3984M 17.2.0 e1d6a67b021e dada74ddbe4c 2024-09-16T08:15:08.475 INFO:teuthology.orchestra.run.smithi028.stdout:prometheus.smithi028 smithi028 *:9095 running (7m) 2m ago 7m 63.2M - 514e6a882f6e fd3df637374d 2024-09-16T08:15:08.831 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:15:08 smithi144 ceph-mon[25668]: pgmap v4: 97 pgs: 97 active+clean; 344 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2024-09-16T08:15:08.831 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:15:08 smithi144 ceph-mon[25668]: Deploying cephadm binary to smithi028 2024-09-16T08:15:08.831 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:15:08 smithi144 ceph-mon[25668]: from='client.14700 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T08:15:08.831 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:15:08 smithi144 ceph-mon[25668]: [16/Sep/2024:08:15:07] ENGINE Bus STARTING 2024-09-16T08:15:08.832 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:15:08 smithi144 ceph-mon[25668]: [16/Sep/2024:08:15:07] ENGINE Serving on http://172.21.15.144:8765 2024-09-16T08:15:09.010 INFO:teuthology.orchestra.run.smithi028.stdout: 5 72169 12.25 MB/sec execute 7 sec latency 110.210 ms 2024-09-16T08:15:09.308 INFO:teuthology.orchestra.run.smithi028.stdout:{ 2024-09-16T08:15:09.309 INFO:teuthology.orchestra.run.smithi028.stdout: "mon": { 2024-09-16T08:15:09.309 INFO:teuthology.orchestra.run.smithi028.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-16T08:15:09.309 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-09-16T08:15:09.309 INFO:teuthology.orchestra.run.smithi028.stdout: "mgr": { 2024-09-16T08:15:09.309 INFO:teuthology.orchestra.run.smithi028.stdout: "ceph version 19.3.0-4917-g8293d73f (8293d73f8690540e843a81caec373f9cc29cf705) squid (dev)": 1 2024-09-16T08:15:09.309 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-09-16T08:15:09.309 INFO:teuthology.orchestra.run.smithi028.stdout: "osd": { 2024-09-16T08:15:09.309 INFO:teuthology.orchestra.run.smithi028.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-09-16T08:15:09.309 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-09-16T08:15:09.309 INFO:teuthology.orchestra.run.smithi028.stdout: "mds": { 2024-09-16T08:15:09.309 INFO:teuthology.orchestra.run.smithi028.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-16T08:15:09.310 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-09-16T08:15:09.310 INFO:teuthology.orchestra.run.smithi028.stdout: "overall": { 2024-09-16T08:15:09.310 INFO:teuthology.orchestra.run.smithi028.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 12, 2024-09-16T08:15:09.310 INFO:teuthology.orchestra.run.smithi028.stdout: "ceph version 19.3.0-4917-g8293d73f (8293d73f8690540e843a81caec373f9cc29cf705) squid (dev)": 1 2024-09-16T08:15:09.310 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-09-16T08:15:09.310 INFO:teuthology.orchestra.run.smithi028.stdout:} 2024-09-16T08:15:09.434 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:15:09 smithi028 ceph-mon[23744]: [16/Sep/2024:08:15:07] ENGINE Serving on https://172.21.15.144:7150 2024-09-16T08:15:09.434 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:15:09 smithi028 ceph-mon[23744]: [16/Sep/2024:08:15:07] ENGINE Bus STARTED 2024-09-16T08:15:09.434 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:15:09 smithi028 ceph-mon[23744]: [16/Sep/2024:08:15:07] ENGINE Error in HTTPServer.serve 2024-09-16T08:15:09.434 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout: Traceback (most recent call last): 2024-09-16T08:15:09.434 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-16T08:15:09.434 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout: self._connections.run(self.expiration_interval) 2024-09-16T08:15:09.435 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-16T08:15:09.435 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout: self._run(expiration_interval) 2024-09-16T08:15:09.435 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-16T08:15:09.435 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-16T08:15:09.435 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-16T08:15:09.435 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-16T08:15:09.435 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-16T08:15:09.435 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout: s = self.context.wrap_socket( 2024-09-16T08:15:09.435 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-16T08:15:09.435 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout: return self.sslsocket_class._create( 2024-09-16T08:15:09.435 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-16T08:15:09.435 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout: self.do_handshake() 2024-09-16T08:15:09.435 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-16T08:15:09.436 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout: self._sslobj.do_handshake() 2024-09-16T08:15:09.436 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-16T08:15:09.436 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:15:09 smithi028 ceph-mon[23744]: from='client.14704 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T08:15:09.436 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:15:09 smithi028 ceph-mon[23744]: mgrmap e26: smithi144.utzszj(active, since 4s) 2024-09-16T08:15:09.436 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:15:09 smithi028 ceph-mon[23744]: from='client.14708 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T08:15:09.436 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:15:09 smithi028 ceph-mon[23744]: from='client.? 172.21.15.28:0/3158950399' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-16T08:15:09.831 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:15:09 smithi144 ceph-mon[25668]: [16/Sep/2024:08:15:07] ENGINE Serving on https://172.21.15.144:7150 2024-09-16T08:15:09.831 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:15:09 smithi144 ceph-mon[25668]: [16/Sep/2024:08:15:07] ENGINE Bus STARTED 2024-09-16T08:15:09.831 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:15:09 smithi144 ceph-mon[25668]: [16/Sep/2024:08:15:07] ENGINE Error in HTTPServer.serve 2024-09-16T08:15:09.831 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: Traceback (most recent call last): 2024-09-16T08:15:09.832 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-16T08:15:09.832 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: self._connections.run(self.expiration_interval) 2024-09-16T08:15:09.832 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-16T08:15:09.832 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: self._run(expiration_interval) 2024-09-16T08:15:09.832 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-16T08:15:09.832 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-16T08:15:09.832 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-16T08:15:09.832 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-16T08:15:09.832 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-16T08:15:09.832 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: s = self.context.wrap_socket( 2024-09-16T08:15:09.832 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-16T08:15:09.832 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: return self.sslsocket_class._create( 2024-09-16T08:15:09.832 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-16T08:15:09.833 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: self.do_handshake() 2024-09-16T08:15:09.833 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-16T08:15:09.833 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: self._sslobj.do_handshake() 2024-09-16T08:15:09.833 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-16T08:15:09.833 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:15:09 smithi144 ceph-mon[25668]: from='client.14704 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T08:15:09.833 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:15:09 smithi144 ceph-mon[25668]: mgrmap e26: smithi144.utzszj(active, since 4s) 2024-09-16T08:15:09.833 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:15:09 smithi144 ceph-mon[25668]: from='client.14708 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T08:15:09.833 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:15:09 smithi144 ceph-mon[25668]: from='client.? 172.21.15.28:0/3158950399' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-16T08:15:09.874 INFO:teuthology.orchestra.run.smithi028.stdout:{ 2024-09-16T08:15:09.875 INFO:teuthology.orchestra.run.smithi028.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:f7ad55f9ccd17ba66aadb19b4d291e67b60ea24e5d8a0c0e92941fd59c81e4ce", 2024-09-16T08:15:09.875 INFO:teuthology.orchestra.run.smithi028.stdout: "in_progress": true, 2024-09-16T08:15:09.875 INFO:teuthology.orchestra.run.smithi028.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-16T08:15:09.875 INFO:teuthology.orchestra.run.smithi028.stdout: "services_complete": [], 2024-09-16T08:15:09.875 INFO:teuthology.orchestra.run.smithi028.stdout: "progress": "1/23 daemons upgraded", 2024-09-16T08:15:09.875 INFO:teuthology.orchestra.run.smithi028.stdout: "message": "", 2024-09-16T08:15:09.875 INFO:teuthology.orchestra.run.smithi028.stdout: "is_paused": false 2024-09-16T08:15:09.875 INFO:teuthology.orchestra.run.smithi028.stdout:} 2024-09-16T08:15:10.009 INFO:teuthology.orchestra.run.smithi028.stdout: 5 72562 11.47 MB/sec execute 8 sec latency 97.876 ms 2024-09-16T08:15:10.445 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:15:10 smithi028 ceph-mon[23744]: pgmap v5: 97 pgs: 97 active+clean; 344 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2024-09-16T08:15:10.529 INFO:teuthology.orchestra.run.smithi028.stdout:HEALTH_OK 2024-09-16T08:15:10.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:15:10 smithi144 ceph-mon[25668]: pgmap v5: 97 pgs: 97 active+clean; 344 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2024-09-16T08:15:11.010 INFO:teuthology.orchestra.run.smithi028.stdout: 5 73023 11.59 MB/sec execute 9 sec latency 43.709 ms 2024-09-16T08:15:11.457 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:15:11 smithi028 ceph-mon[23744]: from='client.14716 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T08:15:11.457 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:15:11 smithi028 ceph-mon[23744]: from='client.? 172.21.15.28:0/2937950879' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-16T08:15:11.831 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:15:11 smithi144 ceph-mon[25668]: from='client.14716 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T08:15:11.831 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:15:11 smithi144 ceph-mon[25668]: from='client.? 172.21.15.28:0/2937950879' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-16T08:15:12.010 INFO:teuthology.orchestra.run.smithi028.stdout: 5 73615 11.95 MB/sec execute 10 sec latency 50.461 ms 2024-09-16T08:15:12.483 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:15:12 smithi028 ceph-mon[23744]: pgmap v6: 97 pgs: 97 active+clean; 344 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2024-09-16T08:15:12.831 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:15:12 smithi144 ceph-mon[25668]: pgmap v6: 97 pgs: 97 active+clean; 344 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2024-09-16T08:15:13.010 INFO:teuthology.orchestra.run.smithi028.stdout: 5 74006 12.08 MB/sec execute 11 sec latency 81.104 ms 2024-09-16T08:15:14.010 INFO:teuthology.orchestra.run.smithi028.stdout: 5 74511 12.75 MB/sec execute 12 sec latency 45.856 ms 2024-09-16T08:15:14.514 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:15:14 smithi028 ceph-mon[23744]: pgmap v7: 97 pgs: 97 active+clean; 330 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 33 KiB/s rd, 3.8 MiB/s wr, 452 op/s 2024-09-16T08:15:14.514 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:15:14 smithi028 ceph-mon[23744]: Standby manager daemon smithi028.azvyzy started 2024-09-16T08:15:14.514 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:15:14 smithi028 ceph-mon[23744]: from='mgr.? 172.21.15.28:0/4285764072' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi028.azvyzy/crt"}]: dispatch 2024-09-16T08:15:14.514 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:15:14 smithi028 ceph-mon[23744]: from='mgr.? 172.21.15.28:0/4285764072' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-16T08:15:14.514 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:15:14 smithi028 ceph-mon[23744]: from='mgr.? 172.21.15.28:0/4285764072' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi028.azvyzy/key"}]: dispatch 2024-09-16T08:15:14.514 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:15:14 smithi028 ceph-mon[23744]: from='mgr.? 172.21.15.28:0/4285764072' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-16T08:15:14.831 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:15:14 smithi144 ceph-mon[25668]: pgmap v7: 97 pgs: 97 active+clean; 330 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 33 KiB/s rd, 3.8 MiB/s wr, 452 op/s 2024-09-16T08:15:14.832 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:15:14 smithi144 ceph-mon[25668]: Standby manager daemon smithi028.azvyzy started 2024-09-16T08:15:14.832 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:15:14 smithi144 ceph-mon[25668]: from='mgr.? 172.21.15.28:0/4285764072' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi028.azvyzy/crt"}]: dispatch 2024-09-16T08:15:14.832 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:15:14 smithi144 ceph-mon[25668]: from='mgr.? 172.21.15.28:0/4285764072' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-16T08:15:14.832 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:15:14 smithi144 ceph-mon[25668]: from='mgr.? 172.21.15.28:0/4285764072' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi028.azvyzy/key"}]: dispatch 2024-09-16T08:15:14.832 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:15:14 smithi144 ceph-mon[25668]: from='mgr.? 172.21.15.28:0/4285764072' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-16T08:15:15.010 INFO:teuthology.orchestra.run.smithi028.stdout: 5 75062 13.03 MB/sec execute 13 sec latency 43.604 ms 2024-09-16T08:15:15.518 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:15:15 smithi028 ceph-mon[23744]: from='mgr.24459 172.21.15.144:0/4176577021' entity='mgr.smithi144.utzszj' cmd=[{"prefix": "mgr metadata", "who": "smithi028.azvyzy", "id": "smithi028.azvyzy"}]: dispatch 2024-09-16T08:15:15.518 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:15:15 smithi028 ceph-mon[23744]: mgrmap e27: smithi144.utzszj(active, since 10s), standbys: smithi028.azvyzy 2024-09-16T08:15:15.518 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:15:15 smithi028 ceph-mon[23744]: from='mgr.24459 ' entity='mgr.smithi144.utzszj' 2024-09-16T08:15:15.518 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:15:15 smithi028 ceph-mon[23744]: from='mgr.24459 ' entity='mgr.smithi144.utzszj' 2024-09-16T08:15:15.831 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:15:15 smithi144 ceph-mon[25668]: from='mgr.24459 172.21.15.144:0/4176577021' entity='mgr.smithi144.utzszj' cmd=[{"prefix": "mgr metadata", "who": "smithi028.azvyzy", "id": "smithi028.azvyzy"}]: dispatch 2024-09-16T08:15:15.831 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:15:15 smithi144 ceph-mon[25668]: mgrmap e27: smithi144.utzszj(active, since 10s), standbys: smithi028.azvyzy 2024-09-16T08:15:15.831 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:15:15 smithi144 ceph-mon[25668]: from='mgr.24459 ' entity='mgr.smithi144.utzszj' 2024-09-16T08:15:15.831 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:15:15 smithi144 ceph-mon[25668]: from='mgr.24459 ' entity='mgr.smithi144.utzszj' 2024-09-16T08:15:16.010 INFO:teuthology.orchestra.run.smithi028.stdout: 5 75382 12.58 MB/sec execute 14 sec latency 45.301 ms 2024-09-16T08:15:16.535 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:15:16 smithi028 ceph-mon[23744]: pgmap v8: 97 pgs: 97 active+clean; 330 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 26 KiB/s rd, 2.9 MiB/s wr, 351 op/s 2024-09-16T08:15:16.535 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:15:16 smithi028 ceph-mon[23744]: from='mgr.24459 ' entity='mgr.smithi144.utzszj' 2024-09-16T08:15:16.535 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:15:16 smithi028 ceph-mon[23744]: from='mgr.24459 ' entity='mgr.smithi144.utzszj' 2024-09-16T08:15:16.535 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:15:16 smithi028 ceph-mon[23744]: from='mgr.24459 172.21.15.144:0/4176577021' entity='mgr.smithi144.utzszj' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-16T08:15:16.535 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:15:16 smithi028 ceph-mon[23744]: from='mgr.24459 ' entity='mgr.smithi144.utzszj' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-16T08:15:16.535 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:15:16 smithi028 ceph-mon[23744]: from='mgr.24459 172.21.15.144:0/4176577021' entity='mgr.smithi144.utzszj' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-16T08:15:16.535 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:15:16 smithi028 ceph-mon[23744]: from='mgr.24459 ' entity='mgr.smithi144.utzszj' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-16T08:15:16.535 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:15:16 smithi028 ceph-mon[23744]: from='mgr.24459 172.21.15.144:0/4176577021' entity='mgr.smithi144.utzszj' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-16T08:15:16.536 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:15:16 smithi028 ceph-mon[23744]: from='mgr.24459 ' entity='mgr.smithi144.utzszj' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-16T08:15:16.536 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:15:16 smithi028 ceph-mon[23744]: from='mgr.24459 172.21.15.144:0/4176577021' entity='mgr.smithi144.utzszj' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-16T08:15:16.536 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:15:16 smithi028 ceph-mon[23744]: from='mgr.24459 ' entity='mgr.smithi144.utzszj' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-16T08:15:16.536 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:15:16 smithi028 ceph-mon[23744]: from='mgr.24459 ' entity='mgr.smithi144.utzszj' 2024-09-16T08:15:16.831 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:15:16 smithi144 ceph-mon[25668]: pgmap v8: 97 pgs: 97 active+clean; 330 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 26 KiB/s rd, 2.9 MiB/s wr, 351 op/s 2024-09-16T08:15:16.831 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:15:16 smithi144 ceph-mon[25668]: from='mgr.24459 ' entity='mgr.smithi144.utzszj' 2024-09-16T08:15:16.831 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:15:16 smithi144 ceph-mon[25668]: from='mgr.24459 ' entity='mgr.smithi144.utzszj' 2024-09-16T08:15:16.832 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:15:16 smithi144 ceph-mon[25668]: from='mgr.24459 172.21.15.144:0/4176577021' entity='mgr.smithi144.utzszj' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-16T08:15:16.832 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:15:16 smithi144 ceph-mon[25668]: from='mgr.24459 ' entity='mgr.smithi144.utzszj' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-16T08:15:16.832 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:15:16 smithi144 ceph-mon[25668]: from='mgr.24459 172.21.15.144:0/4176577021' entity='mgr.smithi144.utzszj' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-16T08:15:16.832 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:15:16 smithi144 ceph-mon[25668]: from='mgr.24459 ' entity='mgr.smithi144.utzszj' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-16T08:15:16.832 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:15:16 smithi144 ceph-mon[25668]: from='mgr.24459 172.21.15.144:0/4176577021' entity='mgr.smithi144.utzszj' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-16T08:15:16.832 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:15:16 smithi144 ceph-mon[25668]: from='mgr.24459 ' entity='mgr.smithi144.utzszj' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-16T08:15:16.832 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:15:16 smithi144 ceph-mon[25668]: from='mgr.24459 172.21.15.144:0/4176577021' entity='mgr.smithi144.utzszj' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-16T08:15:16.832 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:15:16 smithi144 ceph-mon[25668]: from='mgr.24459 ' entity='mgr.smithi144.utzszj' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-16T08:15:16.832 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:15:16 smithi144 ceph-mon[25668]: from='mgr.24459 ' entity='mgr.smithi144.utzszj' 2024-09-16T08:15:17.010 INFO:teuthology.orchestra.run.smithi028.stdout: 5 75733 12.08 MB/sec execute 15 sec latency 93.890 ms 2024-09-16T08:15:17.551 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:15:17 smithi028 ceph-mon[23744]: Adjusting osd_memory_target on smithi028 to 1937M 2024-09-16T08:15:17.798 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:15:17 smithi144 ceph-mon[25668]: Adjusting osd_memory_target on smithi028 to 1937M 2024-09-16T08:15:18.012 INFO:teuthology.orchestra.run.smithi028.stdout: 5 76045 11.66 MB/sec execute 16 sec latency 120.424 ms 2024-09-16T08:15:18.576 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:15:18 smithi028 ceph-mon[23744]: pgmap v9: 97 pgs: 97 active+clean; 338 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 24 KiB/s rd, 6.0 MiB/s wr, 584 op/s 2024-09-16T08:15:18.736 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:15:18 smithi144 ceph-mon[25668]: pgmap v9: 97 pgs: 97 active+clean; 338 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 24 KiB/s rd, 6.0 MiB/s wr, 584 op/s 2024-09-16T08:15:19.010 INFO:teuthology.orchestra.run.smithi028.stdout: 5 76327 11.50 MB/sec execute 17 sec latency 131.625 ms 2024-09-16T08:15:20.010 INFO:teuthology.orchestra.run.smithi028.stdout: 5 76749 11.36 MB/sec execute 18 sec latency 91.131 ms 2024-09-16T08:15:20.108 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:15:19 smithi028 ceph-mon[23744]: pgmap v10: 97 pgs: 97 active+clean; 338 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 22 KiB/s rd, 5.4 MiB/s wr, 532 op/s 2024-09-16T08:15:20.108 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:15:19 smithi028 ceph-mon[23744]: from='mgr.24459 ' entity='mgr.smithi144.utzszj' 2024-09-16T08:15:20.108 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:15:19 smithi028 ceph-mon[23744]: from='mgr.24459 ' entity='mgr.smithi144.utzszj' 2024-09-16T08:15:20.108 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:15:19 smithi028 ceph-mon[23744]: from='mgr.24459 ' entity='mgr.smithi144.utzszj' 2024-09-16T08:15:20.108 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:15:19 smithi028 ceph-mon[23744]: from='mgr.24459 ' entity='mgr.smithi144.utzszj' 2024-09-16T08:15:20.247 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:15:19 smithi144 ceph-mon[25668]: pgmap v10: 97 pgs: 97 active+clean; 338 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 22 KiB/s rd, 5.4 MiB/s wr, 532 op/s 2024-09-16T08:15:20.247 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:15:19 smithi144 ceph-mon[25668]: from='mgr.24459 ' entity='mgr.smithi144.utzszj' 2024-09-16T08:15:20.247 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:15:19 smithi144 ceph-mon[25668]: from='mgr.24459 ' entity='mgr.smithi144.utzszj' 2024-09-16T08:15:20.247 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:15:19 smithi144 ceph-mon[25668]: from='mgr.24459 ' entity='mgr.smithi144.utzszj' 2024-09-16T08:15:20.247 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:15:19 smithi144 ceph-mon[25668]: from='mgr.24459 ' entity='mgr.smithi144.utzszj' 2024-09-16T08:15:21.010 INFO:teuthology.orchestra.run.smithi028.stdout: 5 77287 11.46 MB/sec execute 19 sec latency 43.158 ms 2024-09-16T08:15:21.385 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:15:21 smithi028 ceph-mon[23744]: from='mgr.24459 172.21.15.144:0/4176577021' entity='mgr.smithi144.utzszj' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-16T08:15:21.407 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:15:21 smithi144 ceph-mon[25668]: from='mgr.24459 172.21.15.144:0/4176577021' entity='mgr.smithi144.utzszj' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-16T08:15:22.010 INFO:teuthology.orchestra.run.smithi028.stdout: 5 77700 11.92 MB/sec execute 20 sec latency 89.542 ms 2024-09-16T08:15:22.173 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:15:22 smithi028 ceph-mon[23744]: pgmap v11: 97 pgs: 97 active+clean; 338 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 22 KiB/s rd, 5.4 MiB/s wr, 532 op/s 2024-09-16T08:15:22.270 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:15:22 smithi144 ceph-mon[25668]: pgmap v11: 97 pgs: 97 active+clean; 338 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 22 KiB/s rd, 5.4 MiB/s wr, 532 op/s 2024-09-16T08:15:23.010 INFO:teuthology.orchestra.run.smithi028.stdout: 5 78027 11.80 MB/sec execute 21 sec latency 78.543 ms 2024-09-16T08:15:24.010 INFO:teuthology.orchestra.run.smithi028.stdout: 5 78374 11.84 MB/sec execute 22 sec latency 47.704 ms 2024-09-16T08:15:24.246 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:15:24 smithi028 ceph-mon[23744]: pgmap v12: 97 pgs: 97 active+clean; 323 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 26 KiB/s rd, 7.4 MiB/s wr, 755 op/s 2024-09-16T08:15:24.246 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:15:24 smithi028 ceph-mon[23744]: from='mgr.24459 ' entity='mgr.smithi144.utzszj' 2024-09-16T08:15:24.246 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:15:24 smithi028 ceph-mon[23744]: from='mgr.24459 ' entity='mgr.smithi144.utzszj' 2024-09-16T08:15:24.246 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:15:24 smithi028 ceph-mon[23744]: from='mgr.24459 172.21.15.144:0/4176577021' entity='mgr.smithi144.utzszj' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-16T08:15:24.246 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:15:24 smithi028 ceph-mon[23744]: from='mgr.24459 ' entity='mgr.smithi144.utzszj' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-16T08:15:24.246 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:15:24 smithi028 ceph-mon[23744]: from='mgr.24459 172.21.15.144:0/4176577021' entity='mgr.smithi144.utzszj' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-16T08:15:24.246 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:15:24 smithi028 ceph-mon[23744]: from='mgr.24459 ' entity='mgr.smithi144.utzszj' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-16T08:15:24.247 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:15:24 smithi028 ceph-mon[23744]: from='mgr.24459 172.21.15.144:0/4176577021' entity='mgr.smithi144.utzszj' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-16T08:15:24.247 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:15:24 smithi028 ceph-mon[23744]: from='mgr.24459 ' entity='mgr.smithi144.utzszj' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-16T08:15:24.247 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:15:24 smithi028 ceph-mon[23744]: from='mgr.24459 172.21.15.144:0/4176577021' entity='mgr.smithi144.utzszj' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-16T08:15:24.247 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:15:24 smithi028 ceph-mon[23744]: from='mgr.24459 ' entity='mgr.smithi144.utzszj' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-16T08:15:24.247 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:15:24 smithi028 ceph-mon[23744]: from='mgr.24459 ' entity='mgr.smithi144.utzszj' 2024-09-16T08:15:24.247 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:15:24 smithi028 ceph-mon[23744]: from='mgr.24459 172.21.15.144:0/4176577021' entity='mgr.smithi144.utzszj' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T08:15:24.247 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:15:24 smithi028 ceph-mon[23744]: from='mgr.24459 172.21.15.144:0/4176577021' entity='mgr.smithi144.utzszj' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-16T08:15:24.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:15:24 smithi144 ceph-mon[25668]: pgmap v12: 97 pgs: 97 active+clean; 323 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 26 KiB/s rd, 7.4 MiB/s wr, 755 op/s 2024-09-16T08:15:24.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:15:24 smithi144 ceph-mon[25668]: from='mgr.24459 ' entity='mgr.smithi144.utzszj' 2024-09-16T08:15:24.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:15:24 smithi144 ceph-mon[25668]: from='mgr.24459 ' entity='mgr.smithi144.utzszj' 2024-09-16T08:15:24.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:15:24 smithi144 ceph-mon[25668]: from='mgr.24459 172.21.15.144:0/4176577021' entity='mgr.smithi144.utzszj' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-16T08:15:24.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:15:24 smithi144 ceph-mon[25668]: from='mgr.24459 ' entity='mgr.smithi144.utzszj' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-16T08:15:24.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:15:24 smithi144 ceph-mon[25668]: from='mgr.24459 172.21.15.144:0/4176577021' entity='mgr.smithi144.utzszj' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-16T08:15:24.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:15:24 smithi144 ceph-mon[25668]: from='mgr.24459 ' entity='mgr.smithi144.utzszj' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-16T08:15:24.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:15:24 smithi144 ceph-mon[25668]: from='mgr.24459 172.21.15.144:0/4176577021' entity='mgr.smithi144.utzszj' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-16T08:15:24.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:15:24 smithi144 ceph-mon[25668]: from='mgr.24459 ' entity='mgr.smithi144.utzszj' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-16T08:15:24.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:15:24 smithi144 ceph-mon[25668]: from='mgr.24459 172.21.15.144:0/4176577021' entity='mgr.smithi144.utzszj' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-16T08:15:24.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:15:24 smithi144 ceph-mon[25668]: from='mgr.24459 ' entity='mgr.smithi144.utzszj' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-16T08:15:24.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:15:24 smithi144 ceph-mon[25668]: from='mgr.24459 ' entity='mgr.smithi144.utzszj' 2024-09-16T08:15:24.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:15:24 smithi144 ceph-mon[25668]: from='mgr.24459 172.21.15.144:0/4176577021' entity='mgr.smithi144.utzszj' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T08:15:24.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:15:24 smithi144 ceph-mon[25668]: from='mgr.24459 172.21.15.144:0/4176577021' entity='mgr.smithi144.utzszj' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-16T08:15:25.010 INFO:teuthology.orchestra.run.smithi028.stdout: 5 78681 11.58 MB/sec execute 23 sec latency 46.128 ms 2024-09-16T08:15:25.517 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:15:25 smithi028 ceph-mon[23744]: Adjusting osd_memory_target on smithi144 to 2704M 2024-09-16T08:15:25.517 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:15:25 smithi028 ceph-mon[23744]: Updating smithi028:/etc/ceph/ceph.conf 2024-09-16T08:15:25.517 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:15:25 smithi028 ceph-mon[23744]: Updating smithi144:/etc/ceph/ceph.conf 2024-09-16T08:15:25.518 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:15:25 smithi028 ceph-mon[23744]: Updating smithi144:/var/lib/ceph/5622b2fe-7402-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-16T08:15:25.518 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:15:25 smithi028 ceph-mon[23744]: Updating smithi028:/var/lib/ceph/5622b2fe-7402-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-16T08:15:25.518 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:15:25 smithi028 ceph-mon[23744]: Updating smithi144:/etc/ceph/ceph.client.admin.keyring 2024-09-16T08:15:25.518 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:15:25 smithi028 ceph-mon[23744]: Updating smithi028:/etc/ceph/ceph.client.admin.keyring 2024-09-16T08:15:25.518 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:15:25 smithi028 ceph-mon[23744]: Updating smithi144:/var/lib/ceph/5622b2fe-7402-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-16T08:15:25.518 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:15:25 smithi028 ceph-mon[23744]: Updating smithi028:/var/lib/ceph/5622b2fe-7402-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-16T08:15:25.518 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:15:25 smithi028 ceph-mon[23744]: from='mgr.24459 ' entity='mgr.smithi144.utzszj' 2024-09-16T08:15:25.518 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:15:25 smithi028 ceph-mon[23744]: from='mgr.24459 ' entity='mgr.smithi144.utzszj' 2024-09-16T08:15:25.518 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:15:25 smithi028 ceph-mon[23744]: from='mgr.24459 ' entity='mgr.smithi144.utzszj' 2024-09-16T08:15:25.518 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:15:25 smithi028 ceph-mon[23744]: from='mgr.24459 ' entity='mgr.smithi144.utzszj' 2024-09-16T08:15:25.518 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:15:25 smithi028 ceph-mon[23744]: from='mgr.24459 ' entity='mgr.smithi144.utzszj' 2024-09-16T08:15:25.518 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:15:25 smithi028 ceph-mon[23744]: from='mgr.24459 ' entity='mgr.smithi144.utzszj' 2024-09-16T08:15:25.519 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:15:25 smithi028 ceph-mon[23744]: from='mgr.24459 ' entity='mgr.smithi144.utzszj' 2024-09-16T08:15:25.519 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:15:25 smithi028 ceph-mon[23744]: from='mgr.24459 ' entity='mgr.smithi144.utzszj' 2024-09-16T08:15:25.519 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:15:25 smithi028 ceph-mon[23744]: from='mgr.24459 ' entity='mgr.smithi144.utzszj' 2024-09-16T08:15:25.519 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:15:25 smithi028 ceph-mon[23744]: from='mgr.24459 ' entity='mgr.smithi144.utzszj' 2024-09-16T08:15:25.519 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:15:25 smithi028 ceph-mon[23744]: Reconfiguring alertmanager.smithi028 (dependencies changed)... 2024-09-16T08:15:25.519 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:15:25 smithi028 ceph-mon[23744]: Deploying daemon alertmanager.smithi028 on smithi028 2024-09-16T08:15:25.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:15:25 smithi144 ceph-mon[25668]: Adjusting osd_memory_target on smithi144 to 2704M 2024-09-16T08:15:25.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:15:25 smithi144 ceph-mon[25668]: Updating smithi028:/etc/ceph/ceph.conf 2024-09-16T08:15:25.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:15:25 smithi144 ceph-mon[25668]: Updating smithi144:/etc/ceph/ceph.conf 2024-09-16T08:15:25.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:15:25 smithi144 ceph-mon[25668]: Updating smithi144:/var/lib/ceph/5622b2fe-7402-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-16T08:15:25.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:15:25 smithi144 ceph-mon[25668]: Updating smithi028:/var/lib/ceph/5622b2fe-7402-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-16T08:15:25.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:15:25 smithi144 ceph-mon[25668]: Updating smithi144:/etc/ceph/ceph.client.admin.keyring 2024-09-16T08:15:25.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:15:25 smithi144 ceph-mon[25668]: Updating smithi028:/etc/ceph/ceph.client.admin.keyring 2024-09-16T08:15:25.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:15:25 smithi144 ceph-mon[25668]: Updating smithi144:/var/lib/ceph/5622b2fe-7402-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-16T08:15:25.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:15:25 smithi144 ceph-mon[25668]: Updating smithi028:/var/lib/ceph/5622b2fe-7402-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-16T08:15:25.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:15:25 smithi144 ceph-mon[25668]: from='mgr.24459 ' entity='mgr.smithi144.utzszj' 2024-09-16T08:15:25.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:15:25 smithi144 ceph-mon[25668]: from='mgr.24459 ' entity='mgr.smithi144.utzszj' 2024-09-16T08:15:25.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:15:25 smithi144 ceph-mon[25668]: from='mgr.24459 ' entity='mgr.smithi144.utzszj' 2024-09-16T08:15:25.583 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:15:25 smithi144 ceph-mon[25668]: from='mgr.24459 ' entity='mgr.smithi144.utzszj' 2024-09-16T08:15:25.583 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:15:25 smithi144 ceph-mon[25668]: from='mgr.24459 ' entity='mgr.smithi144.utzszj' 2024-09-16T08:15:25.583 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:15:25 smithi144 ceph-mon[25668]: from='mgr.24459 ' entity='mgr.smithi144.utzszj' 2024-09-16T08:15:25.583 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:15:25 smithi144 ceph-mon[25668]: from='mgr.24459 ' entity='mgr.smithi144.utzszj' 2024-09-16T08:15:25.583 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:15:25 smithi144 ceph-mon[25668]: from='mgr.24459 ' entity='mgr.smithi144.utzszj' 2024-09-16T08:15:25.583 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:15:25 smithi144 ceph-mon[25668]: from='mgr.24459 ' entity='mgr.smithi144.utzszj' 2024-09-16T08:15:25.583 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:15:25 smithi144 ceph-mon[25668]: from='mgr.24459 ' entity='mgr.smithi144.utzszj' 2024-09-16T08:15:25.583 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:15:25 smithi144 ceph-mon[25668]: Reconfiguring alertmanager.smithi028 (dependencies changed)... 2024-09-16T08:15:25.583 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:15:25 smithi144 ceph-mon[25668]: Deploying daemon alertmanager.smithi028 on smithi028 2024-09-16T08:15:26.010 INFO:teuthology.orchestra.run.smithi028.stdout: 5 79075 11.36 MB/sec execute 24 sec latency 55.895 ms 2024-09-16T08:15:26.522 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:15:26 smithi028 ceph-mon[23744]: pgmap v13: 97 pgs: 97 active+clean; 323 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 6.9 KiB/s rd, 5.2 MiB/s wr, 493 op/s 2024-09-16T08:15:26.831 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:15:26 smithi144 ceph-mon[25668]: pgmap v13: 97 pgs: 97 active+clean; 323 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 6.9 KiB/s rd, 5.2 MiB/s wr, 493 op/s 2024-09-16T08:15:27.010 INFO:teuthology.orchestra.run.smithi028.stdout: 5 79553 11.24 MB/sec execute 25 sec latency 32.448 ms 2024-09-16T08:15:28.010 INFO:teuthology.orchestra.run.smithi028.stdout: 5 80021 11.35 MB/sec execute 26 sec latency 25.858 ms 2024-09-16T08:15:28.546 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:15:28 smithi028 ceph-mon[23744]: pgmap v14: 97 pgs: 97 active+clean; 332 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 9.7 KiB/s rd, 7.7 MiB/s wr, 712 op/s 2024-09-16T08:15:28.831 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:15:28 smithi144 ceph-mon[25668]: pgmap v14: 97 pgs: 97 active+clean; 332 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 9.7 KiB/s rd, 7.7 MiB/s wr, 712 op/s 2024-09-16T08:15:29.010 INFO:teuthology.orchestra.run.smithi028.stdout: 5 80662 11.45 MB/sec execute 27 sec latency 38.236 ms 2024-09-16T08:15:30.010 INFO:teuthology.orchestra.run.smithi028.stdout: 5 81155 11.71 MB/sec execute 28 sec latency 22.435 ms 2024-09-16T08:15:30.581 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:15:30 smithi028 ceph-mon[23744]: pgmap v15: 97 pgs: 97 active+clean; 332 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 7.1 KiB/s rd, 4.5 MiB/s wr, 441 op/s 2024-09-16T08:15:30.832 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:15:30 smithi144 ceph-mon[25668]: pgmap v15: 97 pgs: 97 active+clean; 332 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 7.1 KiB/s rd, 4.5 MiB/s wr, 441 op/s 2024-09-16T08:15:31.010 INFO:teuthology.orchestra.run.smithi028.stdout: 5 81777 12.15 MB/sec execute 29 sec latency 31.615 ms 2024-09-16T08:15:32.010 INFO:teuthology.orchestra.run.smithi028.stdout: 5 82050 11.96 MB/sec execute 30 sec latency 70.014 ms 2024-09-16T08:15:32.615 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:15:32 smithi028 ceph-mon[23744]: pgmap v16: 97 pgs: 97 active+clean; 332 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 7.1 KiB/s rd, 4.5 MiB/s wr, 441 op/s 2024-09-16T08:15:32.831 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:15:32 smithi144 ceph-mon[25668]: pgmap v16: 97 pgs: 97 active+clean; 332 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 7.1 KiB/s rd, 4.5 MiB/s wr, 441 op/s 2024-09-16T08:15:33.010 INFO:teuthology.orchestra.run.smithi028.stdout: 5 82235 11.69 MB/sec execute 31 sec latency 58.183 ms 2024-09-16T08:15:34.010 INFO:teuthology.orchestra.run.smithi028.stdout: 5 82617 11.50 MB/sec execute 32 sec latency 34.574 ms 2024-09-16T08:15:34.708 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:15:34 smithi028 ceph-mon[23744]: pgmap v17: 97 pgs: 97 active+clean; 330 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 7.8 MiB/s wr, 757 op/s 2024-09-16T08:15:34.831 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:15:34 smithi144 ceph-mon[25668]: pgmap v17: 97 pgs: 97 active+clean; 330 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 7.8 MiB/s wr, 757 op/s 2024-09-16T08:15:35.010 INFO:teuthology.orchestra.run.smithi028.stdout: 5 83100 11.43 MB/sec execute 33 sec latency 50.498 ms 2024-09-16T08:15:36.011 INFO:teuthology.orchestra.run.smithi028.stdout: 5 83806 11.64 MB/sec execute 34 sec latency 23.400 ms 2024-09-16T08:15:36.235 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:15:36 smithi028 ceph-mon[23744]: pgmap v18: 97 pgs: 97 active+clean; 330 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 6.9 KiB/s rd, 5.9 MiB/s wr, 535 op/s 2024-09-16T08:15:36.235 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:15:36 smithi028 ceph-mon[23744]: from='mgr.24459 172.21.15.144:0/4176577021' entity='mgr.smithi144.utzszj' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-16T08:15:36.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:15:36 smithi144 ceph-mon[25668]: pgmap v18: 97 pgs: 97 active+clean; 330 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 6.9 KiB/s rd, 5.9 MiB/s wr, 535 op/s 2024-09-16T08:15:36.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:15:36 smithi144 ceph-mon[25668]: from='mgr.24459 172.21.15.144:0/4176577021' entity='mgr.smithi144.utzszj' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-16T08:15:37.011 INFO:teuthology.orchestra.run.smithi028.stdout: 5 84447 11.79 MB/sec execute 35 sec latency 41.804 ms 2024-09-16T08:15:38.011 INFO:teuthology.orchestra.run.smithi028.stdout: 5 84957 12.10 MB/sec execute 36 sec latency 48.116 ms 2024-09-16T08:15:38.278 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:15:38 smithi028 ceph-mon[23744]: pgmap v19: 97 pgs: 97 active+clean; 326 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 8.0 MiB/s wr, 787 op/s 2024-09-16T08:15:38.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:15:38 smithi144 ceph-mon[25668]: pgmap v19: 97 pgs: 97 active+clean; 326 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 8.0 MiB/s wr, 787 op/s 2024-09-16T08:15:39.011 INFO:teuthology.orchestra.run.smithi028.stdout: 5 85585 12.24 MB/sec execute 37 sec latency 29.396 ms 2024-09-16T08:15:40.011 INFO:teuthology.orchestra.run.smithi028.stdout: 5 86147 12.27 MB/sec execute 38 sec latency 32.263 ms 2024-09-16T08:15:40.292 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:15:40 smithi028 ceph-mon[23744]: pgmap v20: 97 pgs: 97 active+clean; 326 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 8.4 KiB/s rd, 5.4 MiB/s wr, 568 op/s 2024-09-16T08:15:40.503 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:15:40 smithi144 ceph-mon[25668]: pgmap v20: 97 pgs: 97 active+clean; 326 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 8.4 KiB/s rd, 5.4 MiB/s wr, 568 op/s 2024-09-16T08:15:41.011 INFO:teuthology.orchestra.run.smithi028.stdout: 5 86711 12.15 MB/sec execute 39 sec latency 28.635 ms 2024-09-16T08:15:41.179 INFO:teuthology.orchestra.run.smithi028.stdout:true 2024-09-16T08:15:41.815 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:15:41 smithi028 ceph-mon[23744]: from='mgr.24459 ' entity='mgr.smithi144.utzszj' 2024-09-16T08:15:41.815 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:15:41 smithi028 ceph-mon[23744]: from='mgr.24459 ' entity='mgr.smithi144.utzszj' 2024-09-16T08:15:41.815 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:15:41 smithi028 ceph-mon[23744]: Reconfiguring grafana.smithi028 (dependencies changed)... 2024-09-16T08:15:41.815 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:15:41 smithi028 ceph-mon[23744]: Regenerating cephadm self-signed grafana TLS certificates 2024-09-16T08:15:41.815 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:15:41 smithi028 ceph-mon[23744]: pgmap v21: 97 pgs: 97 active+clean; 326 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 8.4 KiB/s rd, 5.4 MiB/s wr, 568 op/s 2024-09-16T08:15:41.815 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:15:41 smithi028 ceph-mon[23744]: from='client.24513 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T08:15:41.815 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:15:41 smithi028 ceph-mon[23744]: from='mgr.24459 ' entity='mgr.smithi144.utzszj' 2024-09-16T08:15:41.815 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:15:41 smithi028 ceph-mon[23744]: from='mgr.24459 ' entity='mgr.smithi144.utzszj' 2024-09-16T08:15:41.815 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:15:41 smithi028 ceph-mon[23744]: from='mgr.24459 172.21.15.144:0/4176577021' entity='mgr.smithi144.utzszj' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-16T08:15:41.816 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:15:41 smithi028 ceph-mon[23744]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-16T08:15:41.816 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:15:41 smithi028 ceph-mon[23744]: Reconfiguring daemon grafana.smithi028 on smithi028 2024-09-16T08:15:42.011 INFO:teuthology.orchestra.run.smithi028.stdout: 5 87211 12.27 MB/sec execute 40 sec latency 50.207 ms 2024-09-16T08:15:42.081 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:15:41 smithi144 ceph-mon[25668]: from='mgr.24459 ' entity='mgr.smithi144.utzszj' 2024-09-16T08:15:42.081 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:15:41 smithi144 ceph-mon[25668]: from='mgr.24459 ' entity='mgr.smithi144.utzszj' 2024-09-16T08:15:42.081 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:15:41 smithi144 ceph-mon[25668]: Reconfiguring grafana.smithi028 (dependencies changed)... 2024-09-16T08:15:42.081 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:15:41 smithi144 ceph-mon[25668]: Regenerating cephadm self-signed grafana TLS certificates 2024-09-16T08:15:42.081 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:15:41 smithi144 ceph-mon[25668]: pgmap v21: 97 pgs: 97 active+clean; 326 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 8.4 KiB/s rd, 5.4 MiB/s wr, 568 op/s 2024-09-16T08:15:42.082 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:15:41 smithi144 ceph-mon[25668]: from='client.24513 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T08:15:42.082 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:15:41 smithi144 ceph-mon[25668]: from='mgr.24459 ' entity='mgr.smithi144.utzszj' 2024-09-16T08:15:42.082 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:15:41 smithi144 ceph-mon[25668]: from='mgr.24459 ' entity='mgr.smithi144.utzszj' 2024-09-16T08:15:42.082 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:15:41 smithi144 ceph-mon[25668]: from='mgr.24459 172.21.15.144:0/4176577021' entity='mgr.smithi144.utzszj' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-16T08:15:42.082 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:15:41 smithi144 ceph-mon[25668]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-16T08:15:42.082 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:15:41 smithi144 ceph-mon[25668]: Reconfiguring daemon grafana.smithi028 on smithi028 2024-09-16T08:15:42.448 INFO:teuthology.orchestra.run.smithi028.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-16T08:15:42.449 INFO:teuthology.orchestra.run.smithi028.stdout:alertmanager.smithi028 smithi028 *:9093,9094 starting - - - - 2024-09-16T08:15:42.449 INFO:teuthology.orchestra.run.smithi028.stdout:crash.smithi028 smithi028 running (9m) 27s ago 9m 6405k - 17.2.0 e1d6a67b021e 69e795c000fd 2024-09-16T08:15:42.449 INFO:teuthology.orchestra.run.smithi028.stdout:crash.smithi144 smithi144 running (8m) 23s ago 8m 6400k - 17.2.0 e1d6a67b021e bbe54b51649c 2024-09-16T08:15:42.449 INFO:teuthology.orchestra.run.smithi028.stdout:grafana.smithi028 smithi028 *:3000 running (7m) 27s ago 8m 51.9M - dad864ee21e9 6ff027b7efc7 2024-09-16T08:15:42.449 INFO:teuthology.orchestra.run.smithi028.stdout:mds.foofs.smithi028.srtgev smithi028 running (3m) 27s ago 3m 36.4M - 17.2.0 e1d6a67b021e 250e7a7f754c 2024-09-16T08:15:42.449 INFO:teuthology.orchestra.run.smithi028.stdout:mds.foofs.smithi144.oozgsi smithi144 running (3m) 23s ago 3m 15.2M - 17.2.0 e1d6a67b021e ecf4c2bb0fdc 2024-09-16T08:15:42.449 INFO:teuthology.orchestra.run.smithi028.stdout:mgr.smithi028.azvyzy smithi028 *:9283 running (10m) 27s ago 10m 148M - 17.2.0 e1d6a67b021e 229e5e86f252 2024-09-16T08:15:42.449 INFO:teuthology.orchestra.run.smithi028.stdout:mgr.smithi144.utzszj smithi144 *:8443,9283 running (49s) 23s ago 8m 491M - 19.3.0-4917-g8293d73f 425e316ba8eb eecb46cf2442 2024-09-16T08:15:42.449 INFO:teuthology.orchestra.run.smithi028.stdout:mon.smithi028 smithi028 running (10m) 27s ago 10m 53.8M 2048M 17.2.0 e1d6a67b021e a2f688697052 2024-09-16T08:15:42.449 INFO:teuthology.orchestra.run.smithi028.stdout:mon.smithi144 smithi144 running (8m) 23s ago 8m 41.8M 2048M 17.2.0 e1d6a67b021e 1c80e3ff3507 2024-09-16T08:15:42.449 INFO:teuthology.orchestra.run.smithi028.stdout:nfs.foo.0.0.smithi028.dkttgj smithi028 *:2049 running (2m) 27s ago 2m 252M - 4.0 e1d6a67b021e b5dccc45a6ed 2024-09-16T08:15:42.449 INFO:teuthology.orchestra.run.smithi028.stdout:nfs.foo.1.0.smithi144.epipkg smithi144 *:2049 running (2m) 23s ago 2m 77.9M - 4.0 e1d6a67b021e faa89e61f50e 2024-09-16T08:15:42.450 INFO:teuthology.orchestra.run.smithi028.stdout:node-exporter.smithi028 smithi028 *:9100 running (8m) 27s ago 8m 18.0M - 1.3.1 1dbe0e931976 7ad46df29362 2024-09-16T08:15:42.450 INFO:teuthology.orchestra.run.smithi028.stdout:node-exporter.smithi144 smithi144 *:9100 running (8m) 23s ago 8m 18.7M - 1.3.1 1dbe0e931976 f85c6daf2fe0 2024-09-16T08:15:42.450 INFO:teuthology.orchestra.run.smithi028.stdout:osd.0 smithi028 running (7m) 27s ago 7m 293M 1937M 17.2.0 e1d6a67b021e 8184fbc3871f 2024-09-16T08:15:42.450 INFO:teuthology.orchestra.run.smithi028.stdout:osd.1 smithi028 running (7m) 27s ago 7m 236M 1937M 17.2.0 e1d6a67b021e 6911d5c68d54 2024-09-16T08:15:42.450 INFO:teuthology.orchestra.run.smithi028.stdout:osd.2 smithi028 running (6m) 27s ago 6m 187M 1937M 17.2.0 e1d6a67b021e 90b27d8375a2 2024-09-16T08:15:42.450 INFO:teuthology.orchestra.run.smithi028.stdout:osd.3 smithi028 running (6m) 27s ago 6m 259M 1937M 17.2.0 e1d6a67b021e dac477d8b04f 2024-09-16T08:15:42.450 INFO:teuthology.orchestra.run.smithi028.stdout:osd.4 smithi144 running (5m) 23s ago 5m 256M 2704M 17.2.0 e1d6a67b021e 86719c60975c 2024-09-16T08:15:42.450 INFO:teuthology.orchestra.run.smithi028.stdout:osd.5 smithi144 running (5m) 23s ago 5m 292M 2704M 17.2.0 e1d6a67b021e 70452c145c60 2024-09-16T08:15:42.450 INFO:teuthology.orchestra.run.smithi028.stdout:osd.6 smithi144 running (5m) 23s ago 5m 265M 2704M 17.2.0 e1d6a67b021e 7f9981be792f 2024-09-16T08:15:42.450 INFO:teuthology.orchestra.run.smithi028.stdout:osd.7 smithi144 running (4m) 23s ago 4m 371M 2704M 17.2.0 e1d6a67b021e dada74ddbe4c 2024-09-16T08:15:42.450 INFO:teuthology.orchestra.run.smithi028.stdout:prometheus.smithi028 smithi028 *:9095 running (8m) 27s ago 8m 65.4M - 2.33.4 514e6a882f6e fd3df637374d 2024-09-16T08:15:42.818 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:15:42 smithi028 ceph-mon[23744]: from='client.24517 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T08:15:42.819 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:15:42 smithi028 ceph-mon[23744]: from='client.24521 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T08:15:43.011 INFO:teuthology.orchestra.run.smithi028.stdout: 5 87810 12.24 MB/sec execute 41 sec latency 49.527 ms 2024-09-16T08:15:43.081 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:15:42 smithi144 ceph-mon[25668]: from='client.24517 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T08:15:43.081 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:15:42 smithi144 ceph-mon[25668]: from='client.24521 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T08:15:43.220 INFO:teuthology.orchestra.run.smithi028.stdout:{ 2024-09-16T08:15:43.221 INFO:teuthology.orchestra.run.smithi028.stdout: "mon": { 2024-09-16T08:15:43.221 INFO:teuthology.orchestra.run.smithi028.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-16T08:15:43.221 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-09-16T08:15:43.221 INFO:teuthology.orchestra.run.smithi028.stdout: "mgr": { 2024-09-16T08:15:43.221 INFO:teuthology.orchestra.run.smithi028.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 1, 2024-09-16T08:15:43.221 INFO:teuthology.orchestra.run.smithi028.stdout: "ceph version 19.3.0-4917-g8293d73f (8293d73f8690540e843a81caec373f9cc29cf705) squid (dev)": 1 2024-09-16T08:15:43.221 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-09-16T08:15:43.221 INFO:teuthology.orchestra.run.smithi028.stdout: "osd": { 2024-09-16T08:15:43.221 INFO:teuthology.orchestra.run.smithi028.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-09-16T08:15:43.221 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-09-16T08:15:43.221 INFO:teuthology.orchestra.run.smithi028.stdout: "mds": { 2024-09-16T08:15:43.222 INFO:teuthology.orchestra.run.smithi028.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-16T08:15:43.222 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-09-16T08:15:43.222 INFO:teuthology.orchestra.run.smithi028.stdout: "overall": { 2024-09-16T08:15:43.222 INFO:teuthology.orchestra.run.smithi028.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 13, 2024-09-16T08:15:43.222 INFO:teuthology.orchestra.run.smithi028.stdout: "ceph version 19.3.0-4917-g8293d73f (8293d73f8690540e843a81caec373f9cc29cf705) squid (dev)": 1 2024-09-16T08:15:43.222 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-09-16T08:15:43.222 INFO:teuthology.orchestra.run.smithi028.stdout:} 2024-09-16T08:15:43.826 INFO:teuthology.orchestra.run.smithi028.stdout:{ 2024-09-16T08:15:43.826 INFO:teuthology.orchestra.run.smithi028.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:f7ad55f9ccd17ba66aadb19b4d291e67b60ea24e5d8a0c0e92941fd59c81e4ce", 2024-09-16T08:15:43.826 INFO:teuthology.orchestra.run.smithi028.stdout: "in_progress": true, 2024-09-16T08:15:43.826 INFO:teuthology.orchestra.run.smithi028.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-16T08:15:43.827 INFO:teuthology.orchestra.run.smithi028.stdout: "services_complete": [], 2024-09-16T08:15:43.827 INFO:teuthology.orchestra.run.smithi028.stdout: "progress": "1/23 daemons upgraded", 2024-09-16T08:15:43.827 INFO:teuthology.orchestra.run.smithi028.stdout: "message": "", 2024-09-16T08:15:43.827 INFO:teuthology.orchestra.run.smithi028.stdout: "is_paused": false 2024-09-16T08:15:43.827 INFO:teuthology.orchestra.run.smithi028.stdout:} 2024-09-16T08:15:43.833 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:15:43 smithi028 ceph-mon[23744]: pgmap v22: 97 pgs: 97 active+clean; 333 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 9.2 MiB/s wr, 907 op/s 2024-09-16T08:15:43.834 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:15:43 smithi028 ceph-mon[23744]: from='client.? 172.21.15.28:0/2881418709' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-16T08:15:44.011 INFO:teuthology.orchestra.run.smithi028.stdout: 5 88209 12.29 MB/sec execute 42 sec latency 29.541 ms 2024-09-16T08:15:44.081 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:15:43 smithi144 ceph-mon[25668]: pgmap v22: 97 pgs: 97 active+clean; 333 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 9.2 MiB/s wr, 907 op/s 2024-09-16T08:15:44.081 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:15:43 smithi144 ceph-mon[25668]: from='client.? 172.21.15.28:0/2881418709' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-16T08:15:44.575 INFO:teuthology.orchestra.run.smithi028.stdout:HEALTH_OK 2024-09-16T08:15:44.842 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:15:44 smithi028 ceph-mon[23744]: from='client.14736 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T08:15:44.842 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:15:44 smithi028 ceph-mon[23744]: from='client.? 172.21.15.28:0/1948239156' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-16T08:15:45.011 INFO:teuthology.orchestra.run.smithi028.stdout: 5 88717 12.46 MB/sec execute 43 sec latency 35.450 ms 2024-09-16T08:15:45.081 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:15:44 smithi144 ceph-mon[25668]: from='client.14736 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T08:15:45.081 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:15:44 smithi144 ceph-mon[25668]: from='client.? 172.21.15.28:0/1948239156' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-16T08:15:46.011 INFO:teuthology.orchestra.run.smithi028.stdout: 5 89174 12.51 MB/sec execute 44 sec latency 26.245 ms 2024-09-16T08:15:46.148 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:15:46 smithi028 ceph-mon[23744]: pgmap v23: 97 pgs: 97 active+clean; 333 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 8.2 KiB/s rd, 5.8 MiB/s wr, 590 op/s 2024-09-16T08:15:46.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:15:46 smithi144 ceph-mon[25668]: pgmap v23: 97 pgs: 97 active+clean; 333 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 8.2 KiB/s rd, 5.8 MiB/s wr, 590 op/s 2024-09-16T08:15:47.011 INFO:teuthology.orchestra.run.smithi028.stdout: 5 89506 12.41 MB/sec execute 45 sec latency 26.569 ms 2024-09-16T08:15:48.011 INFO:teuthology.orchestra.run.smithi028.stdout: 5 89949 12.26 MB/sec execute 46 sec latency 58.318 ms 2024-09-16T08:15:48.196 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:15:48 smithi028 ceph-mon[23744]: pgmap v24: 97 pgs: 97 active+clean; 333 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 13 KiB/s rd, 8.9 MiB/s wr, 876 op/s 2024-09-16T08:15:48.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:15:48 smithi144 ceph-mon[25668]: pgmap v24: 97 pgs: 97 active+clean; 333 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 13 KiB/s rd, 8.9 MiB/s wr, 876 op/s 2024-09-16T08:15:49.011 INFO:teuthology.orchestra.run.smithi028.stdout: 5 90340 12.15 MB/sec execute 47 sec latency 134.855 ms 2024-09-16T08:15:50.012 INFO:teuthology.orchestra.run.smithi028.stdout: 5 90819 12.23 MB/sec execute 48 sec latency 35.235 ms 2024-09-16T08:15:50.212 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:15:50 smithi028 ceph-mon[23744]: pgmap v25: 97 pgs: 97 active+clean; 333 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 8.5 KiB/s rd, 6.8 MiB/s wr, 624 op/s 2024-09-16T08:15:50.213 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:15:50 smithi028 ceph-mon[23744]: from='mgr.24459 ' entity='mgr.smithi144.utzszj' 2024-09-16T08:15:50.213 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:15:50 smithi028 ceph-mon[23744]: from='mgr.24459 ' entity='mgr.smithi144.utzszj' 2024-09-16T08:15:50.213 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:15:50 smithi028 ceph-mon[23744]: Reconfiguring node-exporter.smithi028 (dependencies changed)... 2024-09-16T08:15:50.213 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:15:50 smithi028 ceph-mon[23744]: Deploying daemon node-exporter.smithi028 on smithi028 2024-09-16T08:15:50.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:15:50 smithi144 ceph-mon[25668]: pgmap v25: 97 pgs: 97 active+clean; 333 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 8.5 KiB/s rd, 6.8 MiB/s wr, 624 op/s 2024-09-16T08:15:50.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:15:50 smithi144 ceph-mon[25668]: from='mgr.24459 ' entity='mgr.smithi144.utzszj' 2024-09-16T08:15:50.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:15:50 smithi144 ceph-mon[25668]: from='mgr.24459 ' entity='mgr.smithi144.utzszj' 2024-09-16T08:15:50.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:15:50 smithi144 ceph-mon[25668]: Reconfiguring node-exporter.smithi028 (dependencies changed)... 2024-09-16T08:15:50.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:15:50 smithi144 ceph-mon[25668]: Deploying daemon node-exporter.smithi028 on smithi028 2024-09-16T08:15:51.012 INFO:teuthology.orchestra.run.smithi028.stdout: 5 91555 12.34 MB/sec execute 49 sec latency 24.587 ms 2024-09-16T08:15:51.223 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:15:51 smithi028 ceph-mon[23744]: from='mgr.24459 172.21.15.144:0/4176577021' entity='mgr.smithi144.utzszj' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-16T08:15:51.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:15:51 smithi144 ceph-mon[25668]: from='mgr.24459 172.21.15.144:0/4176577021' entity='mgr.smithi144.utzszj' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-16T08:15:52.012 INFO:teuthology.orchestra.run.smithi028.stdout: 5 92041 12.51 MB/sec execute 50 sec latency 25.545 ms 2024-09-16T08:15:52.234 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:15:52 smithi028 ceph-mon[23744]: pgmap v26: 97 pgs: 97 active+clean; 333 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 8.5 KiB/s rd, 6.8 MiB/s wr, 624 op/s 2024-09-16T08:15:52.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:15:52 smithi144 ceph-mon[25668]: pgmap v26: 97 pgs: 97 active+clean; 333 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 8.5 KiB/s rd, 6.8 MiB/s wr, 624 op/s 2024-09-16T08:15:53.012 INFO:teuthology.orchestra.run.smithi028.stdout: 5 92728 12.66 MB/sec execute 51 sec latency 35.140 ms 2024-09-16T08:15:54.012 INFO:teuthology.orchestra.run.smithi028.stdout: 5 93349 12.66 MB/sec execute 52 sec latency 19.860 ms 2024-09-16T08:15:54.244 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:15:54 smithi028 ceph-mon[23744]: pgmap v27: 97 pgs: 97 active+clean; 326 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 13 KiB/s rd, 9.5 MiB/s wr, 906 op/s 2024-09-16T08:15:54.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:15:54 smithi144 ceph-mon[25668]: pgmap v27: 97 pgs: 97 active+clean; 326 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 13 KiB/s rd, 9.5 MiB/s wr, 906 op/s 2024-09-16T08:15:55.012 INFO:teuthology.orchestra.run.smithi028.stdout: 5 93890 12.58 MB/sec execute 53 sec latency 38.873 ms 2024-09-16T08:15:56.012 INFO:teuthology.orchestra.run.smithi028.stdout: 5 94554 12.74 MB/sec execute 54 sec latency 32.311 ms 2024-09-16T08:15:56.273 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:15:56 smithi028 ceph-mon[23744]: pgmap v28: 97 pgs: 97 active+clean; 326 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 9.2 KiB/s rd, 5.8 MiB/s wr, 567 op/s 2024-09-16T08:15:56.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:15:56 smithi144 ceph-mon[25668]: pgmap v28: 97 pgs: 97 active+clean; 326 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 9.2 KiB/s rd, 5.8 MiB/s wr, 567 op/s 2024-09-16T08:15:57.012 INFO:teuthology.orchestra.run.smithi028.stdout: 5 95141 12.78 MB/sec execute 55 sec latency 47.103 ms 2024-09-16T08:15:57.802 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:15:57 smithi028 ceph-mon[23744]: from='mgr.24459 ' entity='mgr.smithi144.utzszj' 2024-09-16T08:15:57.802 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:15:57 smithi028 ceph-mon[23744]: from='mgr.24459 ' entity='mgr.smithi144.utzszj' 2024-09-16T08:15:57.802 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:15:57 smithi028 ceph-mon[23744]: Reconfiguring prometheus.smithi028 (dependencies changed)... 2024-09-16T08:15:57.802 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:15:57 smithi028 ceph-mon[23744]: pgmap v29: 97 pgs: 97 active+clean; 335 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 14 KiB/s rd, 9.4 MiB/s wr, 921 op/s 2024-09-16T08:15:57.802 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:15:57 smithi028 ceph-mon[23744]: Deploying daemon prometheus.smithi028 on smithi028 2024-09-16T08:15:58.012 INFO:teuthology.orchestra.run.smithi028.stdout: 5 95625 12.90 MB/sec execute 56 sec latency 25.611 ms 2024-09-16T08:15:58.081 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:15:57 smithi144 ceph-mon[25668]: from='mgr.24459 ' entity='mgr.smithi144.utzszj' 2024-09-16T08:15:58.081 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:15:57 smithi144 ceph-mon[25668]: from='mgr.24459 ' entity='mgr.smithi144.utzszj' 2024-09-16T08:15:58.081 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:15:57 smithi144 ceph-mon[25668]: Reconfiguring prometheus.smithi028 (dependencies changed)... 2024-09-16T08:15:58.082 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:15:57 smithi144 ceph-mon[25668]: pgmap v29: 97 pgs: 97 active+clean; 335 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 14 KiB/s rd, 9.4 MiB/s wr, 921 op/s 2024-09-16T08:15:58.082 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:15:57 smithi144 ceph-mon[25668]: Deploying daemon prometheus.smithi028 on smithi028 2024-09-16T08:15:59.013 INFO:teuthology.orchestra.run.smithi028.stdout: 5 96087 12.90 MB/sec execute 57 sec latency 51.573 ms 2024-09-16T08:16:00.013 INFO:teuthology.orchestra.run.smithi028.stdout: 5 96505 12.87 MB/sec execute 58 sec latency 33.981 ms 2024-09-16T08:16:00.063 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:15:59 smithi028 ceph-mon[23744]: pgmap v30: 97 pgs: 97 active+clean; 335 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 9.1 KiB/s rd, 6.3 MiB/s wr, 636 op/s 2024-09-16T08:16:00.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:15:59 smithi144 ceph-mon[25668]: pgmap v30: 97 pgs: 97 active+clean; 335 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 9.1 KiB/s rd, 6.3 MiB/s wr, 636 op/s 2024-09-16T08:16:01.013 INFO:teuthology.orchestra.run.smithi028.stdout: 5 97135 12.81 MB/sec execute 59 sec latency 21.583 ms 2024-09-16T08:16:02.013 INFO:teuthology.orchestra.run.smithi028.stdout: 5 97601 12.78 MB/sec execute 60 sec latency 30.578 ms 2024-09-16T08:16:02.355 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:16:02 smithi028 ceph-mon[23744]: pgmap v31: 97 pgs: 97 active+clean; 335 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 9.1 KiB/s rd, 6.3 MiB/s wr, 636 op/s 2024-09-16T08:16:02.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:16:02 smithi144 ceph-mon[25668]: pgmap v31: 97 pgs: 97 active+clean; 335 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 9.1 KiB/s rd, 6.3 MiB/s wr, 636 op/s 2024-09-16T08:16:03.012 INFO:teuthology.orchestra.run.smithi028.stdout: 5 98129 12.84 MB/sec execute 61 sec latency 63.078 ms 2024-09-16T08:16:04.013 INFO:teuthology.orchestra.run.smithi028.stdout: 5 98551 12.76 MB/sec execute 62 sec latency 60.802 ms 2024-09-16T08:16:04.391 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:16:04 smithi028 ceph-mon[23744]: pgmap v32: 97 pgs: 97 active+clean; 335 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 13 KiB/s rd, 9.6 MiB/s wr, 940 op/s 2024-09-16T08:16:04.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:16:04 smithi144 ceph-mon[25668]: pgmap v32: 97 pgs: 97 active+clean; 335 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 13 KiB/s rd, 9.6 MiB/s wr, 940 op/s 2024-09-16T08:16:05.013 INFO:teuthology.orchestra.run.smithi028.stdout: 5 98817 12.76 MB/sec execute 63 sec latency 53.971 ms 2024-09-16T08:16:06.013 INFO:teuthology.orchestra.run.smithi028.stdout: 5 99124 12.77 MB/sec execute 64 sec latency 88.279 ms 2024-09-16T08:16:06.240 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:16:06 smithi028 ceph-mon[23744]: pgmap v33: 97 pgs: 97 active+clean; 335 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 8.5 KiB/s rd, 7.0 MiB/s wr, 657 op/s 2024-09-16T08:16:06.240 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:16:06 smithi028 ceph-mon[23744]: from='mgr.24459 172.21.15.144:0/4176577021' entity='mgr.smithi144.utzszj' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-16T08:16:06.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:16:06 smithi144 ceph-mon[25668]: pgmap v33: 97 pgs: 97 active+clean; 335 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 8.5 KiB/s rd, 7.0 MiB/s wr, 657 op/s 2024-09-16T08:16:06.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:16:06 smithi144 ceph-mon[25668]: from='mgr.24459 172.21.15.144:0/4176577021' entity='mgr.smithi144.utzszj' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-16T08:16:07.013 INFO:teuthology.orchestra.run.smithi028.stdout: 5 99446 12.71 MB/sec execute 65 sec latency 95.025 ms 2024-09-16T08:16:08.013 INFO:teuthology.orchestra.run.smithi028.stdout: 5 99686 12.61 MB/sec execute 66 sec latency 98.570 ms 2024-09-16T08:16:08.204 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:16:08 smithi028 ceph-mon[23744]: pgmap v34: 97 pgs: 97 active+clean; 328 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 13 KiB/s rd, 9.5 MiB/s wr, 906 op/s 2024-09-16T08:16:08.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:16:08 smithi144 ceph-mon[25668]: pgmap v34: 97 pgs: 97 active+clean; 328 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 13 KiB/s rd, 9.5 MiB/s wr, 906 op/s 2024-09-16T08:16:09.013 INFO:teuthology.orchestra.run.smithi028.stdout: 5 99901 12.51 MB/sec execute 67 sec latency 53.402 ms 2024-09-16T08:16:10.013 INFO:teuthology.orchestra.run.smithi028.stdout: 5 100052 12.38 MB/sec execute 68 sec latency 102.915 ms 2024-09-16T08:16:10.479 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:16:10 smithi028 ceph-mon[23744]: pgmap v35: 97 pgs: 97 active+clean; 328 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 8.4 KiB/s rd, 5.9 MiB/s wr, 552 op/s 2024-09-16T08:16:10.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:16:10 smithi144 ceph-mon[25668]: pgmap v35: 97 pgs: 97 active+clean; 328 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 8.4 KiB/s rd, 5.9 MiB/s wr, 552 op/s 2024-09-16T08:16:11.013 INFO:teuthology.orchestra.run.smithi028.stdout: 5 100383 12.31 MB/sec execute 69 sec latency 48.476 ms 2024-09-16T08:16:12.013 INFO:teuthology.orchestra.run.smithi028.stdout: 5 100738 12.17 MB/sec execute 70 sec latency 88.469 ms 2024-09-16T08:16:12.285 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:16:12 smithi028 ceph-mon[23744]: pgmap v36: 97 pgs: 97 active+clean; 328 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 8.4 KiB/s rd, 5.9 MiB/s wr, 552 op/s 2024-09-16T08:16:12.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:16:12 smithi144 ceph-mon[25668]: pgmap v36: 97 pgs: 97 active+clean; 328 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 8.4 KiB/s rd, 5.9 MiB/s wr, 552 op/s 2024-09-16T08:16:13.013 INFO:teuthology.orchestra.run.smithi028.stdout: 5 100966 12.07 MB/sec execute 71 sec latency 57.173 ms 2024-09-16T08:16:14.014 INFO:teuthology.orchestra.run.smithi028.stdout: 5 101234 12.03 MB/sec execute 72 sec latency 112.378 ms 2024-09-16T08:16:14.316 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:16:14 smithi028 ceph-mon[23744]: pgmap v37: 97 pgs: 97 active+clean; 329 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 7.6 MiB/s wr, 712 op/s 2024-09-16T08:16:14.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:16:14 smithi144 ceph-mon[25668]: pgmap v37: 97 pgs: 97 active+clean; 329 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 7.6 MiB/s wr, 712 op/s 2024-09-16T08:16:15.013 INFO:teuthology.orchestra.run.smithi028.stdout: 5 101544 12.02 MB/sec execute 73 sec latency 67.836 ms 2024-09-16T08:16:15.062 INFO:teuthology.orchestra.run.smithi028.stdout:true 2024-09-16T08:16:16.013 INFO:teuthology.orchestra.run.smithi028.stdout: 5 101867 11.92 MB/sec execute 74 sec latency 64.929 ms 2024-09-16T08:16:16.110 INFO:teuthology.orchestra.run.smithi028.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-16T08:16:16.110 INFO:teuthology.orchestra.run.smithi028.stdout:alertmanager.smithi028 smithi028 *:9093,9094 starting - - - - 2024-09-16T08:16:16.111 INFO:teuthology.orchestra.run.smithi028.stdout:crash.smithi028 smithi028 running (9m) 61s ago 9m 6405k - 17.2.0 e1d6a67b021e 69e795c000fd 2024-09-16T08:16:16.111 INFO:teuthology.orchestra.run.smithi028.stdout:crash.smithi144 smithi144 running (9m) 57s ago 9m 6400k - 17.2.0 e1d6a67b021e bbe54b51649c 2024-09-16T08:16:16.111 INFO:teuthology.orchestra.run.smithi028.stdout:grafana.smithi028 smithi028 *:3000 starting - - - - 2024-09-16T08:16:16.111 INFO:teuthology.orchestra.run.smithi028.stdout:mds.foofs.smithi028.srtgev smithi028 running (3m) 61s ago 3m 36.4M - 17.2.0 e1d6a67b021e 250e7a7f754c 2024-09-16T08:16:16.111 INFO:teuthology.orchestra.run.smithi028.stdout:mds.foofs.smithi144.oozgsi smithi144 running (3m) 57s ago 3m 15.2M - 17.2.0 e1d6a67b021e ecf4c2bb0fdc 2024-09-16T08:16:16.111 INFO:teuthology.orchestra.run.smithi028.stdout:mgr.smithi028.azvyzy smithi028 *:9283 running (10m) 61s ago 10m 148M - 17.2.0 e1d6a67b021e 229e5e86f252 2024-09-16T08:16:16.111 INFO:teuthology.orchestra.run.smithi028.stdout:mgr.smithi144.utzszj smithi144 *:8443,9283 running (83s) 57s ago 9m 491M - 19.3.0-4917-g8293d73f 425e316ba8eb eecb46cf2442 2024-09-16T08:16:16.111 INFO:teuthology.orchestra.run.smithi028.stdout:mon.smithi028 smithi028 running (10m) 61s ago 10m 53.8M 2048M 17.2.0 e1d6a67b021e a2f688697052 2024-09-16T08:16:16.111 INFO:teuthology.orchestra.run.smithi028.stdout:mon.smithi144 smithi144 running (8m) 57s ago 8m 41.8M 2048M 17.2.0 e1d6a67b021e 1c80e3ff3507 2024-09-16T08:16:16.111 INFO:teuthology.orchestra.run.smithi028.stdout:nfs.foo.0.0.smithi028.dkttgj smithi028 *:2049 running (3m) 61s ago 3m 252M - 4.0 e1d6a67b021e b5dccc45a6ed 2024-09-16T08:16:16.111 INFO:teuthology.orchestra.run.smithi028.stdout:nfs.foo.1.0.smithi144.epipkg smithi144 *:2049 running (3m) 57s ago 3m 77.9M - 4.0 e1d6a67b021e faa89e61f50e 2024-09-16T08:16:16.112 INFO:teuthology.orchestra.run.smithi028.stdout:node-exporter.smithi028 smithi028 *:9100 starting - - - - 2024-09-16T08:16:16.112 INFO:teuthology.orchestra.run.smithi028.stdout:node-exporter.smithi144 smithi144 *:9100 running (8m) 57s ago 8m 18.7M - 1.3.1 1dbe0e931976 f85c6daf2fe0 2024-09-16T08:16:16.112 INFO:teuthology.orchestra.run.smithi028.stdout:osd.0 smithi028 running (8m) 61s ago 8m 293M 1937M 17.2.0 e1d6a67b021e 8184fbc3871f 2024-09-16T08:16:16.112 INFO:teuthology.orchestra.run.smithi028.stdout:osd.1 smithi028 running (7m) 61s ago 7m 236M 1937M 17.2.0 e1d6a67b021e 6911d5c68d54 2024-09-16T08:16:16.112 INFO:teuthology.orchestra.run.smithi028.stdout:osd.2 smithi028 running (7m) 61s ago 7m 187M 1937M 17.2.0 e1d6a67b021e 90b27d8375a2 2024-09-16T08:16:16.112 INFO:teuthology.orchestra.run.smithi028.stdout:osd.3 smithi028 running (6m) 61s ago 6m 259M 1937M 17.2.0 e1d6a67b021e dac477d8b04f 2024-09-16T08:16:16.112 INFO:teuthology.orchestra.run.smithi028.stdout:osd.4 smithi144 running (6m) 57s ago 6m 256M 2704M 17.2.0 e1d6a67b021e 86719c60975c 2024-09-16T08:16:16.112 INFO:teuthology.orchestra.run.smithi028.stdout:osd.5 smithi144 running (6m) 57s ago 6m 292M 2704M 17.2.0 e1d6a67b021e 70452c145c60 2024-09-16T08:16:16.112 INFO:teuthology.orchestra.run.smithi028.stdout:osd.6 smithi144 running (5m) 57s ago 5m 265M 2704M 17.2.0 e1d6a67b021e 7f9981be792f 2024-09-16T08:16:16.112 INFO:teuthology.orchestra.run.smithi028.stdout:osd.7 smithi144 running (5m) 57s ago 5m 371M 2704M 17.2.0 e1d6a67b021e dada74ddbe4c 2024-09-16T08:16:16.113 INFO:teuthology.orchestra.run.smithi028.stdout:prometheus.smithi028 smithi028 *:9095 running (8m) 61s ago 8m 65.4M - 2.33.4 514e6a882f6e fd3df637374d 2024-09-16T08:16:16.420 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:16:16 smithi028 ceph-mon[23744]: pgmap v38: 97 pgs: 97 active+clean; 329 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 7.1 KiB/s rd, 4.3 MiB/s wr, 408 op/s 2024-09-16T08:16:16.420 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:16:16 smithi028 ceph-mon[23744]: from='client.14744 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T08:16:16.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:16:16 smithi144 ceph-mon[25668]: pgmap v38: 97 pgs: 97 active+clean; 329 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 7.1 KiB/s rd, 4.3 MiB/s wr, 408 op/s 2024-09-16T08:16:16.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:16:16 smithi144 ceph-mon[25668]: from='client.14744 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T08:16:16.667 INFO:teuthology.orchestra.run.smithi028.stdout:{ 2024-09-16T08:16:16.668 INFO:teuthology.orchestra.run.smithi028.stdout: "mon": { 2024-09-16T08:16:16.668 INFO:teuthology.orchestra.run.smithi028.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-16T08:16:16.668 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-09-16T08:16:16.668 INFO:teuthology.orchestra.run.smithi028.stdout: "mgr": { 2024-09-16T08:16:16.668 INFO:teuthology.orchestra.run.smithi028.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 1, 2024-09-16T08:16:16.668 INFO:teuthology.orchestra.run.smithi028.stdout: "ceph version 19.3.0-4917-g8293d73f (8293d73f8690540e843a81caec373f9cc29cf705) squid (dev)": 1 2024-09-16T08:16:16.668 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-09-16T08:16:16.668 INFO:teuthology.orchestra.run.smithi028.stdout: "osd": { 2024-09-16T08:16:16.668 INFO:teuthology.orchestra.run.smithi028.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-09-16T08:16:16.668 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-09-16T08:16:16.668 INFO:teuthology.orchestra.run.smithi028.stdout: "mds": { 2024-09-16T08:16:16.669 INFO:teuthology.orchestra.run.smithi028.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-16T08:16:16.669 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-09-16T08:16:16.669 INFO:teuthology.orchestra.run.smithi028.stdout: "overall": { 2024-09-16T08:16:16.669 INFO:teuthology.orchestra.run.smithi028.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 13, 2024-09-16T08:16:16.669 INFO:teuthology.orchestra.run.smithi028.stdout: "ceph version 19.3.0-4917-g8293d73f (8293d73f8690540e843a81caec373f9cc29cf705) squid (dev)": 1 2024-09-16T08:16:16.669 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-09-16T08:16:16.669 INFO:teuthology.orchestra.run.smithi028.stdout:} 2024-09-16T08:16:17.013 INFO:teuthology.orchestra.run.smithi028.stdout: 5 102205 11.89 MB/sec execute 75 sec latency 89.607 ms 2024-09-16T08:16:17.254 INFO:teuthology.orchestra.run.smithi028.stdout:{ 2024-09-16T08:16:17.254 INFO:teuthology.orchestra.run.smithi028.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:f7ad55f9ccd17ba66aadb19b4d291e67b60ea24e5d8a0c0e92941fd59c81e4ce", 2024-09-16T08:16:17.254 INFO:teuthology.orchestra.run.smithi028.stdout: "in_progress": true, 2024-09-16T08:16:17.254 INFO:teuthology.orchestra.run.smithi028.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-16T08:16:17.254 INFO:teuthology.orchestra.run.smithi028.stdout: "services_complete": [], 2024-09-16T08:16:17.254 INFO:teuthology.orchestra.run.smithi028.stdout: "progress": "1/23 daemons upgraded", 2024-09-16T08:16:17.254 INFO:teuthology.orchestra.run.smithi028.stdout: "message": "", 2024-09-16T08:16:17.254 INFO:teuthology.orchestra.run.smithi028.stdout: "is_paused": false 2024-09-16T08:16:17.254 INFO:teuthology.orchestra.run.smithi028.stdout:} 2024-09-16T08:16:17.434 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:16:17 smithi028 ceph-mon[23744]: from='client.14748 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T08:16:17.434 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:16:17 smithi028 ceph-mon[23744]: from='client.14752 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T08:16:17.434 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:16:17 smithi028 ceph-mon[23744]: from='client.? 172.21.15.28:0/1416439229' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-16T08:16:17.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:16:17 smithi144 ceph-mon[25668]: from='client.14748 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T08:16:17.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:16:17 smithi144 ceph-mon[25668]: from='client.14752 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T08:16:17.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:16:17 smithi144 ceph-mon[25668]: from='client.? 172.21.15.28:0/1416439229' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-16T08:16:17.997 INFO:teuthology.orchestra.run.smithi028.stdout:HEALTH_OK 2024-09-16T08:16:18.013 INFO:teuthology.orchestra.run.smithi028.stdout: 5 102619 11.96 MB/sec execute 76 sec latency 34.687 ms 2024-09-16T08:16:18.438 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:16:18 smithi028 ceph-mon[23744]: pgmap v39: 97 pgs: 97 active+clean; 322 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 5.8 MiB/s wr, 589 op/s 2024-09-16T08:16:18.438 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:16:18 smithi028 ceph-mon[23744]: from='client.24545 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T08:16:18.438 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:16:18 smithi028 ceph-mon[23744]: from='client.? 172.21.15.28:0/3138148190' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-16T08:16:18.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:16:18 smithi144 ceph-mon[25668]: pgmap v39: 97 pgs: 97 active+clean; 322 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 5.8 MiB/s wr, 589 op/s 2024-09-16T08:16:18.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:16:18 smithi144 ceph-mon[25668]: from='client.24545 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T08:16:18.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:16:18 smithi144 ceph-mon[25668]: from='client.? 172.21.15.28:0/3138148190' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-16T08:16:19.013 INFO:teuthology.orchestra.run.smithi028.stdout: 5 103144 12.02 MB/sec execute 77 sec latency 25.359 ms 2024-09-16T08:16:19.956 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:16:19 smithi028 ceph-mon[23744]: from='mgr.24459 ' entity='mgr.smithi144.utzszj' 2024-09-16T08:16:19.956 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:16:19 smithi028 ceph-mon[23744]: from='mgr.24459 ' entity='mgr.smithi144.utzszj' 2024-09-16T08:16:19.956 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:16:19 smithi028 ceph-mon[23744]: Reconfiguring node-exporter.smithi144 (dependencies changed)... 2024-09-16T08:16:19.956 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:16:19 smithi028 ceph-mon[23744]: Deploying daemon node-exporter.smithi144 on smithi144 2024-09-16T08:16:19.956 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:16:19 smithi028 ceph-mon[23744]: pgmap v40: 97 pgs: 97 active+clean; 322 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 5.7 KiB/s rd, 3.2 MiB/s wr, 340 op/s 2024-09-16T08:16:20.013 INFO:teuthology.orchestra.run.smithi028.stdout: 5 103440 11.97 MB/sec execute 78 sec latency 49.514 ms 2024-09-16T08:16:20.081 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:16:19 smithi144 ceph-mon[25668]: from='mgr.24459 ' entity='mgr.smithi144.utzszj' 2024-09-16T08:16:20.081 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:16:19 smithi144 ceph-mon[25668]: from='mgr.24459 ' entity='mgr.smithi144.utzszj' 2024-09-16T08:16:20.082 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:16:19 smithi144 ceph-mon[25668]: Reconfiguring node-exporter.smithi144 (dependencies changed)... 2024-09-16T08:16:20.082 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:16:19 smithi144 ceph-mon[25668]: Deploying daemon node-exporter.smithi144 on smithi144 2024-09-16T08:16:20.082 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:16:19 smithi144 ceph-mon[25668]: pgmap v40: 97 pgs: 97 active+clean; 322 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 5.7 KiB/s rd, 3.2 MiB/s wr, 340 op/s 2024-09-16T08:16:21.014 INFO:teuthology.orchestra.run.smithi028.stdout: 5 103700 11.87 MB/sec execute 79 sec latency 72.348 ms 2024-09-16T08:16:21.257 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:16:21 smithi028 ceph-mon[23744]: from='mgr.24459 172.21.15.144:0/4176577021' entity='mgr.smithi144.utzszj' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-16T08:16:21.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:16:21 smithi144 ceph-mon[25668]: from='mgr.24459 172.21.15.144:0/4176577021' entity='mgr.smithi144.utzszj' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-16T08:16:22.013 INFO:teuthology.orchestra.run.smithi028.stdout: 5 104037 11.82 MB/sec execute 80 sec latency 54.012 ms 2024-09-16T08:16:22.273 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:16:22 smithi028 ceph-mon[23744]: pgmap v41: 97 pgs: 97 active+clean; 322 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 5.7 KiB/s rd, 3.2 MiB/s wr, 340 op/s 2024-09-16T08:16:22.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:16:22 smithi144 ceph-mon[25668]: pgmap v41: 97 pgs: 97 active+clean; 322 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 5.7 KiB/s rd, 3.2 MiB/s wr, 340 op/s 2024-09-16T08:16:23.013 INFO:teuthology.orchestra.run.smithi028.stdout: 5 104475 11.75 MB/sec execute 81 sec latency 69.307 ms 2024-09-16T08:16:24.014 INFO:teuthology.orchestra.run.smithi028.stdout: 5 104944 11.78 MB/sec execute 82 sec latency 36.771 ms 2024-09-16T08:16:24.298 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:16:24 smithi028 ceph-mon[23744]: pgmap v42: 97 pgs: 97 active+clean; 335 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 8.1 KiB/s rd, 5.9 MiB/s wr, 562 op/s 2024-09-16T08:16:24.298 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:16:24 smithi028 ceph-mon[23744]: from='mgr.24459 ' entity='mgr.smithi144.utzszj' 2024-09-16T08:16:24.298 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:16:24 smithi028 ceph-mon[23744]: from='mgr.24459 ' entity='mgr.smithi144.utzszj' 2024-09-16T08:16:24.531 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:16:24 smithi144 ceph-mon[25668]: pgmap v42: 97 pgs: 97 active+clean; 335 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 8.1 KiB/s rd, 5.9 MiB/s wr, 562 op/s 2024-09-16T08:16:24.531 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:16:24 smithi144 ceph-mon[25668]: from='mgr.24459 ' entity='mgr.smithi144.utzszj' 2024-09-16T08:16:24.531 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:16:24 smithi144 ceph-mon[25668]: from='mgr.24459 ' entity='mgr.smithi144.utzszj' 2024-09-16T08:16:25.014 INFO:teuthology.orchestra.run.smithi028.stdout: 5 105511 11.81 MB/sec execute 83 sec latency 52.308 ms 2024-09-16T08:16:25.216 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:16:25 smithi028 ceph-mon[23744]: from='mgr.24459 172.21.15.144:0/4176577021' entity='mgr.smithi144.utzszj' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-16T08:16:25.216 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:16:25 smithi028 ceph-mon[23744]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-16T08:16:25.216 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:16:25 smithi028 ceph-mon[23744]: from='mgr.24459 172.21.15.144:0/4176577021' entity='mgr.smithi144.utzszj' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi028.front.sepia.ceph.com:9093"}]: dispatch 2024-09-16T08:16:25.216 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:16:25 smithi028 ceph-mon[23744]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi028.front.sepia.ceph.com:9093"}]: dispatch 2024-09-16T08:16:25.217 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:16:25 smithi028 ceph-mon[23744]: from='mgr.24459 ' entity='mgr.smithi144.utzszj' 2024-09-16T08:16:25.217 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:16:25 smithi028 ceph-mon[23744]: from='mgr.24459 172.21.15.144:0/4176577021' entity='mgr.smithi144.utzszj' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-16T08:16:25.217 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:16:25 smithi028 ceph-mon[23744]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-16T08:16:25.217 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:16:25 smithi028 ceph-mon[23744]: from='mgr.24459 172.21.15.144:0/4176577021' entity='mgr.smithi144.utzszj' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi028.front.sepia.ceph.com:3000"}]: dispatch 2024-09-16T08:16:25.217 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:16:25 smithi028 ceph-mon[23744]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi028.front.sepia.ceph.com:3000"}]: dispatch 2024-09-16T08:16:25.217 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:16:25 smithi028 ceph-mon[23744]: from='mgr.24459 ' entity='mgr.smithi144.utzszj' 2024-09-16T08:16:25.217 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:16:25 smithi028 ceph-mon[23744]: from='mgr.24459 172.21.15.144:0/4176577021' entity='mgr.smithi144.utzszj' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-16T08:16:25.217 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:16:25 smithi028 ceph-mon[23744]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-16T08:16:25.217 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:16:25 smithi028 ceph-mon[23744]: from='mgr.24459 172.21.15.144:0/4176577021' entity='mgr.smithi144.utzszj' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi028.front.sepia.ceph.com:9095"}]: dispatch 2024-09-16T08:16:25.217 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:16:25 smithi028 ceph-mon[23744]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi028.front.sepia.ceph.com:9095"}]: dispatch 2024-09-16T08:16:25.217 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:16:25 smithi028 ceph-mon[23744]: from='mgr.24459 172.21.15.144:0/4176577021' entity='mgr.smithi144.utzszj' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-16T08:16:25.217 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:16:25 smithi028 ceph-mon[23744]: from='mon.? -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-16T08:16:25.217 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:16:25 smithi028 ceph-mon[23744]: from='mgr.24459 ' entity='mgr.smithi144.utzszj' 2024-09-16T08:16:25.217 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:16:25 smithi028 ceph-mon[23744]: from='mgr.24459 172.21.15.144:0/4176577021' entity='mgr.smithi144.utzszj' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-16T08:16:25.218 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:16:25 smithi028 ceph-mon[23744]: Upgrade: Need to upgrade myself (mgr.smithi144.utzszj) 2024-09-16T08:16:25.218 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:16:25 smithi028 ceph-mon[23744]: Upgrade: Need to upgrade myself (mgr.smithi144.utzszj) 2024-09-16T08:16:25.218 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:16:25 smithi028 ceph-mon[23744]: from='mgr.24459 172.21.15.144:0/4176577021' entity='mgr.smithi144.utzszj' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-16T08:16:25.418 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:16:25 smithi144 ceph-mon[25668]: from='mgr.24459 172.21.15.144:0/4176577021' entity='mgr.smithi144.utzszj' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-16T08:16:25.419 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:16:25 smithi144 ceph-mon[25668]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-16T08:16:25.419 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:16:25 smithi144 ceph-mon[25668]: from='mgr.24459 172.21.15.144:0/4176577021' entity='mgr.smithi144.utzszj' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi028.front.sepia.ceph.com:9093"}]: dispatch 2024-09-16T08:16:25.419 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:16:25 smithi144 ceph-mon[25668]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi028.front.sepia.ceph.com:9093"}]: dispatch 2024-09-16T08:16:25.419 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:16:25 smithi144 ceph-mon[25668]: from='mgr.24459 ' entity='mgr.smithi144.utzszj' 2024-09-16T08:16:25.419 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:16:25 smithi144 ceph-mon[25668]: from='mgr.24459 172.21.15.144:0/4176577021' entity='mgr.smithi144.utzszj' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-16T08:16:25.419 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:16:25 smithi144 ceph-mon[25668]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-16T08:16:25.419 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:16:25 smithi144 ceph-mon[25668]: from='mgr.24459 172.21.15.144:0/4176577021' entity='mgr.smithi144.utzszj' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi028.front.sepia.ceph.com:3000"}]: dispatch 2024-09-16T08:16:25.419 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:16:25 smithi144 ceph-mon[25668]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi028.front.sepia.ceph.com:3000"}]: dispatch 2024-09-16T08:16:25.419 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:16:25 smithi144 ceph-mon[25668]: from='mgr.24459 ' entity='mgr.smithi144.utzszj' 2024-09-16T08:16:25.419 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:16:25 smithi144 ceph-mon[25668]: from='mgr.24459 172.21.15.144:0/4176577021' entity='mgr.smithi144.utzszj' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-16T08:16:25.419 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:16:25 smithi144 ceph-mon[25668]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-16T08:16:25.420 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:16:25 smithi144 ceph-mon[25668]: from='mgr.24459 172.21.15.144:0/4176577021' entity='mgr.smithi144.utzszj' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi028.front.sepia.ceph.com:9095"}]: dispatch 2024-09-16T08:16:25.420 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:16:25 smithi144 ceph-mon[25668]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi028.front.sepia.ceph.com:9095"}]: dispatch 2024-09-16T08:16:25.420 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:16:25 smithi144 ceph-mon[25668]: from='mgr.24459 172.21.15.144:0/4176577021' entity='mgr.smithi144.utzszj' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-16T08:16:25.420 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:16:25 smithi144 ceph-mon[25668]: from='mon.? -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-16T08:16:25.420 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:16:25 smithi144 ceph-mon[25668]: from='mgr.24459 ' entity='mgr.smithi144.utzszj' 2024-09-16T08:16:25.420 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:16:25 smithi144 ceph-mon[25668]: from='mgr.24459 172.21.15.144:0/4176577021' entity='mgr.smithi144.utzszj' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-16T08:16:25.420 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:16:25 smithi144 ceph-mon[25668]: Upgrade: Need to upgrade myself (mgr.smithi144.utzszj) 2024-09-16T08:16:25.420 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:16:25 smithi144 ceph-mon[25668]: Upgrade: Need to upgrade myself (mgr.smithi144.utzszj) 2024-09-16T08:16:25.420 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:16:25 smithi144 ceph-mon[25668]: from='mgr.24459 172.21.15.144:0/4176577021' entity='mgr.smithi144.utzszj' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-16T08:16:26.014 INFO:teuthology.orchestra.run.smithi028.stdout: 5 105953 11.86 MB/sec execute 84 sec latency 27.402 ms 2024-09-16T08:16:26.343 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:16:26 smithi028 ceph-mon[23744]: from='mon.? -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-16T08:16:26.343 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:16:26 smithi028 ceph-mon[23744]: pgmap v43: 97 pgs: 97 active+clean; 335 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 5.4 KiB/s rd, 4.2 MiB/s wr, 402 op/s 2024-09-16T08:16:26.343 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:16:26 smithi028 ceph-mon[23744]: from='mgr.24459 172.21.15.144:0/4176577021' entity='mgr.smithi144.utzszj' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-16T08:16:26.343 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:16:26 smithi028 ceph-mon[23744]: from='mon.? -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-16T08:16:26.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:16:26 smithi144 ceph-mon[25668]: from='mon.? -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-16T08:16:26.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:16:26 smithi144 ceph-mon[25668]: pgmap v43: 97 pgs: 97 active+clean; 335 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 5.4 KiB/s rd, 4.2 MiB/s wr, 402 op/s 2024-09-16T08:16:26.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:16:26 smithi144 ceph-mon[25668]: from='mgr.24459 172.21.15.144:0/4176577021' entity='mgr.smithi144.utzszj' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-16T08:16:26.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:16:26 smithi144 ceph-mon[25668]: from='mon.? -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-16T08:16:27.014 INFO:teuthology.orchestra.run.smithi028.stdout: 5 106361 11.90 MB/sec execute 85 sec latency 35.660 ms 2024-09-16T08:16:28.014 INFO:teuthology.orchestra.run.smithi028.stdout: 5 106715 11.89 MB/sec execute 86 sec latency 43.033 ms 2024-09-16T08:16:28.372 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:16:28 smithi028 ceph-mon[23744]: pgmap v44: 97 pgs: 97 active+clean; 326 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 9.6 KiB/s rd, 6.5 MiB/s wr, 666 op/s 2024-09-16T08:16:28.372 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:16:28 smithi028 ceph-mon[23744]: Upgrade: Updating mgr.smithi028.azvyzy 2024-09-16T08:16:28.372 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:16:28 smithi028 ceph-mon[23744]: from='mgr.24459 ' entity='mgr.smithi144.utzszj' 2024-09-16T08:16:28.372 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:16:28 smithi028 ceph-mon[23744]: from='mgr.24459 172.21.15.144:0/4176577021' entity='mgr.smithi144.utzszj' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi028.azvyzy", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-16T08:16:28.372 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:16:28 smithi028 ceph-mon[23744]: from='mgr.24459 ' entity='mgr.smithi144.utzszj' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi028.azvyzy", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-16T08:16:28.372 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:16:28 smithi028 ceph-mon[23744]: from='mgr.24459 172.21.15.144:0/4176577021' entity='mgr.smithi144.utzszj' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-16T08:16:28.372 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:16:28 smithi028 ceph-mon[23744]: from='mgr.24459 172.21.15.144:0/4176577021' entity='mgr.smithi144.utzszj' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T08:16:28.373 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:16:28 smithi028 ceph-mon[23744]: Deploying daemon mgr.smithi028.azvyzy on smithi028 2024-09-16T08:16:28.382 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:16:28 smithi144 ceph-mon[25668]: pgmap v44: 97 pgs: 97 active+clean; 326 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 9.6 KiB/s rd, 6.5 MiB/s wr, 666 op/s 2024-09-16T08:16:28.383 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:16:28 smithi144 ceph-mon[25668]: Upgrade: Updating mgr.smithi028.azvyzy 2024-09-16T08:16:28.383 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:16:28 smithi144 ceph-mon[25668]: from='mgr.24459 ' entity='mgr.smithi144.utzszj' 2024-09-16T08:16:28.383 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:16:28 smithi144 ceph-mon[25668]: from='mgr.24459 172.21.15.144:0/4176577021' entity='mgr.smithi144.utzszj' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi028.azvyzy", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-16T08:16:28.383 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:16:28 smithi144 ceph-mon[25668]: from='mgr.24459 ' entity='mgr.smithi144.utzszj' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi028.azvyzy", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-16T08:16:28.383 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:16:28 smithi144 ceph-mon[25668]: from='mgr.24459 172.21.15.144:0/4176577021' entity='mgr.smithi144.utzszj' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-16T08:16:28.383 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:16:28 smithi144 ceph-mon[25668]: from='mgr.24459 172.21.15.144:0/4176577021' entity='mgr.smithi144.utzszj' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T08:16:28.383 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:16:28 smithi144 ceph-mon[25668]: Deploying daemon mgr.smithi028.azvyzy on smithi028 2024-09-16T08:16:29.014 INFO:teuthology.orchestra.run.smithi028.stdout: 5 107105 11.88 MB/sec execute 87 sec latency 38.476 ms 2024-09-16T08:16:30.016 INFO:teuthology.orchestra.run.smithi028.stdout: 5 107634 11.85 MB/sec execute 88 sec latency 133.830 ms 2024-09-16T08:16:30.133 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:16:30 smithi028 ceph-mon[23744]: pgmap v45: 97 pgs: 97 active+clean; 326 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 6.5 KiB/s rd, 5.0 MiB/s wr, 485 op/s 2024-09-16T08:16:30.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:16:30 smithi144 ceph-mon[25668]: pgmap v45: 97 pgs: 97 active+clean; 326 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 6.5 KiB/s rd, 5.0 MiB/s wr, 485 op/s 2024-09-16T08:16:31.014 INFO:teuthology.orchestra.run.smithi028.stdout: 5 108350 11.88 MB/sec execute 89 sec latency 20.887 ms 2024-09-16T08:16:32.014 INFO:teuthology.orchestra.run.smithi028.stdout: 5 109316 12.06 MB/sec execute 90 sec latency 21.731 ms 2024-09-16T08:16:32.394 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:16:32 smithi028 ceph-mon[23744]: pgmap v46: 97 pgs: 97 active+clean; 326 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 6.5 KiB/s rd, 5.0 MiB/s wr, 485 op/s 2024-09-16T08:16:32.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:16:32 smithi144 ceph-mon[25668]: pgmap v46: 97 pgs: 97 active+clean; 326 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 6.5 KiB/s rd, 5.0 MiB/s wr, 485 op/s 2024-09-16T08:16:33.014 INFO:teuthology.orchestra.run.smithi028.stdout: 5 109797 12.14 MB/sec execute 91 sec latency 66.769 ms 2024-09-16T08:16:34.014 INFO:teuthology.orchestra.run.smithi028.stdout: 5 110286 12.16 MB/sec execute 92 sec latency 47.334 ms 2024-09-16T08:16:34.164 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:16:34 smithi028 ceph-mon[23744]: pgmap v47: 97 pgs: 97 active+clean; 335 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 9.8 KiB/s rd, 8.1 MiB/s wr, 778 op/s 2024-09-16T08:16:34.165 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:16:34 smithi028 ceph-mon[23744]: from='mgr.24459 ' entity='mgr.smithi144.utzszj' 2024-09-16T08:16:34.165 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:16:34 smithi028 ceph-mon[23744]: from='mgr.24459 ' entity='mgr.smithi144.utzszj' 2024-09-16T08:16:34.165 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:16:34 smithi028 ceph-mon[23744]: from='mgr.24459 172.21.15.144:0/4176577021' entity='mgr.smithi144.utzszj' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-16T08:16:34.539 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:16:34 smithi144 ceph-mon[25668]: pgmap v47: 97 pgs: 97 active+clean; 335 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 9.8 KiB/s rd, 8.1 MiB/s wr, 778 op/s 2024-09-16T08:16:34.539 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:16:34 smithi144 ceph-mon[25668]: from='mgr.24459 ' entity='mgr.smithi144.utzszj' 2024-09-16T08:16:34.540 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:16:34 smithi144 ceph-mon[25668]: from='mgr.24459 ' entity='mgr.smithi144.utzszj' 2024-09-16T08:16:34.540 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:16:34 smithi144 ceph-mon[25668]: from='mgr.24459 172.21.15.144:0/4176577021' entity='mgr.smithi144.utzszj' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-16T08:16:35.014 INFO:teuthology.orchestra.run.smithi028.stdout: 5 110588 12.12 MB/sec execute 93 sec latency 34.944 ms 2024-09-16T08:16:35.430 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:16:35 smithi028 ceph-mon[23744]: from='mgr.24459 ' entity='mgr.smithi144.utzszj' 2024-09-16T08:16:35.431 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:16:35 smithi028 ceph-mon[23744]: from='mgr.24459 ' entity='mgr.smithi144.utzszj' 2024-09-16T08:16:35.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:16:35 smithi144 ceph-mon[25668]: from='mgr.24459 ' entity='mgr.smithi144.utzszj' 2024-09-16T08:16:35.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:16:35 smithi144 ceph-mon[25668]: from='mgr.24459 ' entity='mgr.smithi144.utzszj' 2024-09-16T08:16:36.015 INFO:teuthology.orchestra.run.smithi028.stdout: 5 110905 12.08 MB/sec execute 94 sec latency 60.403 ms 2024-09-16T08:16:36.443 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:16:36 smithi028 ceph-mon[23744]: pgmap v48: 97 pgs: 97 active+clean; 335 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 7.5 KiB/s rd, 5.4 MiB/s wr, 556 op/s 2024-09-16T08:16:36.443 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:16:36 smithi028 ceph-mon[23744]: from='mgr.24459 172.21.15.144:0/4176577021' entity='mgr.smithi144.utzszj' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-16T08:16:36.491 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:16:36 smithi144 ceph-mon[25668]: pgmap v48: 97 pgs: 97 active+clean; 335 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 7.5 KiB/s rd, 5.4 MiB/s wr, 556 op/s 2024-09-16T08:16:36.491 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:16:36 smithi144 ceph-mon[25668]: from='mgr.24459 172.21.15.144:0/4176577021' entity='mgr.smithi144.utzszj' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-16T08:16:37.014 INFO:teuthology.orchestra.run.smithi028.stdout: 5 111262 11.98 MB/sec execute 95 sec latency 49.237 ms 2024-09-16T08:16:37.463 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:16:37 smithi028 ceph-mon[23744]: mgrmap e28: smithi144.utzszj(active, since 92s), standbys: smithi028.azvyzy 2024-09-16T08:16:37.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:16:37 smithi144 ceph-mon[25668]: mgrmap e28: smithi144.utzszj(active, since 92s), standbys: smithi028.azvyzy 2024-09-16T08:16:38.023 INFO:teuthology.orchestra.run.smithi028.stdout: 5 111536 11.89 MB/sec execute 96 sec latency 84.497 ms 2024-09-16T08:16:38.328 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:16:38 smithi028 ceph-mon[23744]: pgmap v49: 97 pgs: 97 active+clean; 336 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 8.8 MiB/s wr, 861 op/s 2024-09-16T08:16:38.328 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:16:38 smithi028 ceph-mon[23744]: from='mgr.24459 ' entity='mgr.smithi144.utzszj' 2024-09-16T08:16:38.328 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:16:38 smithi028 ceph-mon[23744]: from='mgr.24459 ' entity='mgr.smithi144.utzszj' 2024-09-16T08:16:38.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:16:38 smithi144 ceph-mon[25668]: pgmap v49: 97 pgs: 97 active+clean; 336 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 8.8 MiB/s wr, 861 op/s 2024-09-16T08:16:38.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:16:38 smithi144 ceph-mon[25668]: from='mgr.24459 ' entity='mgr.smithi144.utzszj' 2024-09-16T08:16:38.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:16:38 smithi144 ceph-mon[25668]: from='mgr.24459 ' entity='mgr.smithi144.utzszj' 2024-09-16T08:16:39.023 INFO:teuthology.orchestra.run.smithi028.stdout: 5 111846 11.88 MB/sec execute 97 sec latency 49.655 ms 2024-09-16T08:16:40.023 INFO:teuthology.orchestra.run.smithi028.stdout: 5 112329 11.91 MB/sec execute 98 sec latency 57.955 ms 2024-09-16T08:16:40.353 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:16:40 smithi028 ceph-mon[23744]: pgmap v50: 97 pgs: 97 active+clean; 336 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 8.3 KiB/s rd, 6.5 MiB/s wr, 597 op/s 2024-09-16T08:16:40.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:16:40 smithi144 ceph-mon[25668]: pgmap v50: 97 pgs: 97 active+clean; 336 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 8.3 KiB/s rd, 6.5 MiB/s wr, 597 op/s 2024-09-16T08:16:41.024 INFO:teuthology.orchestra.run.smithi028.stdout: 5 112771 11.87 MB/sec execute 99 sec latency 32.818 ms 2024-09-16T08:16:41.368 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:16:41 smithi028 ceph-mon[23744]: Standby manager daemon smithi028.azvyzy restarted 2024-09-16T08:16:41.368 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:16:41 smithi028 ceph-mon[23744]: Standby manager daemon smithi028.azvyzy started 2024-09-16T08:16:41.368 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:16:41 smithi028 ceph-mon[23744]: from='mgr.? 172.21.15.28:0/1901954942' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi028.azvyzy/crt"}]: dispatch 2024-09-16T08:16:41.368 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:16:41 smithi028 ceph-mon[23744]: from='mgr.? 172.21.15.28:0/1901954942' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-16T08:16:41.368 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:16:41 smithi028 ceph-mon[23744]: from='mgr.? 172.21.15.28:0/1901954942' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi028.azvyzy/key"}]: dispatch 2024-09-16T08:16:41.368 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:16:41 smithi028 ceph-mon[23744]: from='mgr.? 172.21.15.28:0/1901954942' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-16T08:16:41.586 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:16:41 smithi144 ceph-mon[25668]: Standby manager daemon smithi028.azvyzy restarted 2024-09-16T08:16:41.586 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:16:41 smithi144 ceph-mon[25668]: Standby manager daemon smithi028.azvyzy started 2024-09-16T08:16:41.586 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:16:41 smithi144 ceph-mon[25668]: from='mgr.? 172.21.15.28:0/1901954942' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi028.azvyzy/crt"}]: dispatch 2024-09-16T08:16:41.586 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:16:41 smithi144 ceph-mon[25668]: from='mgr.? 172.21.15.28:0/1901954942' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-16T08:16:41.586 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:16:41 smithi144 ceph-mon[25668]: from='mgr.? 172.21.15.28:0/1901954942' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi028.azvyzy/key"}]: dispatch 2024-09-16T08:16:41.586 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:16:41 smithi144 ceph-mon[25668]: from='mgr.? 172.21.15.28:0/1901954942' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-16T08:16:42.023 INFO:teuthology.orchestra.run.smithi028.stdout: 5 113128 11.91 MB/sec execute 100 sec latency 48.483 ms 2024-09-16T08:16:42.384 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:16:42 smithi028 ceph-mon[23744]: pgmap v51: 97 pgs: 97 active+clean; 336 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 8.3 KiB/s rd, 6.5 MiB/s wr, 597 op/s 2024-09-16T08:16:42.384 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:16:42 smithi028 ceph-mon[23744]: mgrmap e29: smithi144.utzszj(active, since 97s), standbys: smithi028.azvyzy 2024-09-16T08:16:42.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:16:42 smithi144 ceph-mon[25668]: pgmap v51: 97 pgs: 97 active+clean; 336 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 8.3 KiB/s rd, 6.5 MiB/s wr, 597 op/s 2024-09-16T08:16:42.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:16:42 smithi144 ceph-mon[25668]: mgrmap e29: smithi144.utzszj(active, since 97s), standbys: smithi028.azvyzy 2024-09-16T08:16:43.024 INFO:teuthology.orchestra.run.smithi028.stdout: 5 113591 11.96 MB/sec execute 101 sec latency 26.493 ms 2024-09-16T08:16:44.024 INFO:teuthology.orchestra.run.smithi028.stdout: 5 113958 11.94 MB/sec execute 102 sec latency 43.593 ms 2024-09-16T08:16:44.156 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:16:43 smithi028 ceph-mon[23744]: pgmap v52: 97 pgs: 97 active+clean; 325 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 8.4 MiB/s wr, 820 op/s 2024-09-16T08:16:44.156 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:16:43 smithi028 ceph-mon[23744]: from='mgr.24459 ' entity='mgr.smithi144.utzszj' 2024-09-16T08:16:44.156 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:16:43 smithi028 ceph-mon[23744]: from='mgr.24459 ' entity='mgr.smithi144.utzszj' 2024-09-16T08:16:44.156 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:16:43 smithi028 ceph-mon[23744]: from='mgr.24459 ' entity='mgr.smithi144.utzszj' 2024-09-16T08:16:44.157 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:16:43 smithi028 ceph-mon[23744]: from='mgr.24459 ' entity='mgr.smithi144.utzszj' 2024-09-16T08:16:44.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:16:43 smithi144 ceph-mon[25668]: pgmap v52: 97 pgs: 97 active+clean; 325 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 8.4 MiB/s wr, 820 op/s 2024-09-16T08:16:44.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:16:43 smithi144 ceph-mon[25668]: from='mgr.24459 ' entity='mgr.smithi144.utzszj' 2024-09-16T08:16:44.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:16:43 smithi144 ceph-mon[25668]: from='mgr.24459 ' entity='mgr.smithi144.utzszj' 2024-09-16T08:16:44.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:16:43 smithi144 ceph-mon[25668]: from='mgr.24459 ' entity='mgr.smithi144.utzszj' 2024-09-16T08:16:44.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:16:43 smithi144 ceph-mon[25668]: from='mgr.24459 ' entity='mgr.smithi144.utzszj' 2024-09-16T08:16:45.024 INFO:teuthology.orchestra.run.smithi028.stdout: 5 114250 11.89 MB/sec execute 103 sec latency 33.252 ms 2024-09-16T08:16:46.024 INFO:teuthology.orchestra.run.smithi028.stdout: 5 114636 11.85 MB/sec execute 104 sec latency 60.523 ms 2024-09-16T08:16:46.176 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:16:46 smithi028 ceph-mon[23744]: pgmap v53: 97 pgs: 97 active+clean; 325 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 8.7 KiB/s rd, 5.4 MiB/s wr, 527 op/s 2024-09-16T08:16:46.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:16:46 smithi144 ceph-mon[25668]: pgmap v53: 97 pgs: 97 active+clean; 325 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 8.7 KiB/s rd, 5.4 MiB/s wr, 527 op/s 2024-09-16T08:16:47.024 INFO:teuthology.orchestra.run.smithi028.stdout: 5 115207 11.83 MB/sec execute 105 sec latency 32.749 ms 2024-09-16T08:16:48.024 INFO:teuthology.orchestra.run.smithi028.stdout: 5 115483 11.78 MB/sec execute 106 sec latency 63.674 ms 2024-09-16T08:16:48.216 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:16:48 smithi028 ceph-mon[23744]: pgmap v54: 97 pgs: 97 active+clean; 332 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 7.9 MiB/s wr, 762 op/s 2024-09-16T08:16:48.503 INFO:teuthology.orchestra.run.smithi028.stdout:true 2024-09-16T08:16:48.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:16:48 smithi144 ceph-mon[25668]: pgmap v54: 97 pgs: 97 active+clean; 332 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 7.9 MiB/s wr, 762 op/s 2024-09-16T08:16:49.024 INFO:teuthology.orchestra.run.smithi028.stdout: 5 115900 11.80 MB/sec execute 107 sec latency 86.593 ms 2024-09-16T08:16:49.232 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:16:49 smithi028 ceph-mon[23744]: from='client.14772 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T08:16:49.471 INFO:teuthology.orchestra.run.smithi028.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-16T08:16:49.471 INFO:teuthology.orchestra.run.smithi028.stdout:alertmanager.smithi028 smithi028 *:9093,9094 running (69s) 6s ago 10m 19.6M - 0.27.0 11f11916f8cd a14fcb4cb2b6 2024-09-16T08:16:49.471 INFO:teuthology.orchestra.run.smithi028.stdout:crash.smithi028 smithi028 running (10m) 6s ago 10m 6405k - 17.2.0 e1d6a67b021e 69e795c000fd 2024-09-16T08:16:49.471 INFO:teuthology.orchestra.run.smithi028.stdout:crash.smithi144 smithi144 running (9m) 11s ago 9m 6400k - 17.2.0 e1d6a67b021e bbe54b51649c 2024-09-16T08:16:49.471 INFO:teuthology.orchestra.run.smithi028.stdout:grafana.smithi028 smithi028 *:3000 running (61s) 6s ago 9m 45.1M - dad864ee21e9 b55ba8b944a1 2024-09-16T08:16:49.471 INFO:teuthology.orchestra.run.smithi028.stdout:mds.foofs.smithi028.srtgev smithi028 running (4m) 6s ago 4m 36.5M - 17.2.0 e1d6a67b021e 250e7a7f754c 2024-09-16T08:16:49.471 INFO:teuthology.orchestra.run.smithi028.stdout:mds.foofs.smithi144.oozgsi smithi144 running (4m) 11s ago 4m 15.6M - 17.2.0 e1d6a67b021e ecf4c2bb0fdc 2024-09-16T08:16:49.472 INFO:teuthology.orchestra.run.smithi028.stdout:mgr.smithi028.azvyzy smithi028 *:8443,9283,8765 running (16s) 6s ago 11m 86.1M - 19.3.0-4917-g8293d73f 425e316ba8eb e01d4382d03f 2024-09-16T08:16:49.472 INFO:teuthology.orchestra.run.smithi028.stdout:mgr.smithi144.utzszj smithi144 *:8443,9283 running (116s) 11s ago 9m 518M - 19.3.0-4917-g8293d73f 425e316ba8eb eecb46cf2442 2024-09-16T08:16:49.472 INFO:teuthology.orchestra.run.smithi028.stdout:mon.smithi028 smithi028 running (11m) 6s ago 11m 53.8M 2048M 17.2.0 e1d6a67b021e a2f688697052 2024-09-16T08:16:49.472 INFO:teuthology.orchestra.run.smithi028.stdout:mon.smithi144 smithi144 running (9m) 11s ago 9m 43.3M 2048M 17.2.0 e1d6a67b021e 1c80e3ff3507 2024-09-16T08:16:49.472 INFO:teuthology.orchestra.run.smithi028.stdout:nfs.foo.0.0.smithi028.dkttgj smithi028 *:2049 running (4m) 6s ago 4m 276M - 4.0 e1d6a67b021e b5dccc45a6ed 2024-09-16T08:16:49.472 INFO:teuthology.orchestra.run.smithi028.stdout:nfs.foo.1.0.smithi144.epipkg smithi144 *:2049 running (3m) 11s ago 3m 84.3M - 4.0 e1d6a67b021e faa89e61f50e 2024-09-16T08:16:49.472 INFO:teuthology.orchestra.run.smithi028.stdout:node-exporter.smithi028 smithi028 *:9100 running (53s) 6s ago 9m 8409k - 1.7.0 72c9c2088986 9e2b8e8a63a9 2024-09-16T08:16:49.472 INFO:teuthology.orchestra.run.smithi028.stdout:node-exporter.smithi144 smithi144 *:9100 running (26s) 11s ago 9m 7952k - 1.7.0 72c9c2088986 78834c5e992f 2024-09-16T08:16:49.472 INFO:teuthology.orchestra.run.smithi028.stdout:osd.0 smithi028 running (8m) 6s ago 8m 428M 1937M 17.2.0 e1d6a67b021e 8184fbc3871f 2024-09-16T08:16:49.472 INFO:teuthology.orchestra.run.smithi028.stdout:osd.1 smithi028 running (8m) 6s ago 8m 327M 1937M 17.2.0 e1d6a67b021e 6911d5c68d54 2024-09-16T08:16:49.472 INFO:teuthology.orchestra.run.smithi028.stdout:osd.2 smithi028 running (7m) 6s ago 7m 279M 1937M 17.2.0 e1d6a67b021e 90b27d8375a2 2024-09-16T08:16:49.472 INFO:teuthology.orchestra.run.smithi028.stdout:osd.3 smithi028 running (7m) 6s ago 7m 402M 1937M 17.2.0 e1d6a67b021e dac477d8b04f 2024-09-16T08:16:49.472 INFO:teuthology.orchestra.run.smithi028.stdout:osd.4 smithi144 running (7m) 11s ago 7m 371M 2704M 17.2.0 e1d6a67b021e 86719c60975c 2024-09-16T08:16:49.473 INFO:teuthology.orchestra.run.smithi028.stdout:osd.5 smithi144 running (6m) 11s ago 6m 422M 2704M 17.2.0 e1d6a67b021e 70452c145c60 2024-09-16T08:16:49.473 INFO:teuthology.orchestra.run.smithi028.stdout:osd.6 smithi144 running (6m) 11s ago 6m 347M 2704M 17.2.0 e1d6a67b021e 7f9981be792f 2024-09-16T08:16:49.473 INFO:teuthology.orchestra.run.smithi028.stdout:osd.7 smithi144 running (5m) 11s ago 5m 494M 2704M 17.2.0 e1d6a67b021e dada74ddbe4c 2024-09-16T08:16:49.473 INFO:teuthology.orchestra.run.smithi028.stdout:prometheus.smithi028 smithi028 *:9095 running (31s) 6s ago 9m 39.6M - 2.51.0 1d3b7f56885b 1c4dfb787650 2024-09-16T08:16:49.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:16:49 smithi144 ceph-mon[25668]: from='client.14772 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T08:16:50.024 INFO:teuthology.orchestra.run.smithi028.stdout: 5 116234 11.75 MB/sec execute 108 sec latency 77.847 ms 2024-09-16T08:16:50.030 INFO:teuthology.orchestra.run.smithi028.stdout:{ 2024-09-16T08:16:50.030 INFO:teuthology.orchestra.run.smithi028.stdout: "mon": { 2024-09-16T08:16:50.030 INFO:teuthology.orchestra.run.smithi028.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-16T08:16:50.030 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-09-16T08:16:50.030 INFO:teuthology.orchestra.run.smithi028.stdout: "mgr": { 2024-09-16T08:16:50.030 INFO:teuthology.orchestra.run.smithi028.stdout: "ceph version 19.3.0-4917-g8293d73f (8293d73f8690540e843a81caec373f9cc29cf705) squid (dev)": 2 2024-09-16T08:16:50.030 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-09-16T08:16:50.030 INFO:teuthology.orchestra.run.smithi028.stdout: "osd": { 2024-09-16T08:16:50.031 INFO:teuthology.orchestra.run.smithi028.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-09-16T08:16:50.031 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-09-16T08:16:50.031 INFO:teuthology.orchestra.run.smithi028.stdout: "mds": { 2024-09-16T08:16:50.031 INFO:teuthology.orchestra.run.smithi028.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-16T08:16:50.031 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-09-16T08:16:50.031 INFO:teuthology.orchestra.run.smithi028.stdout: "overall": { 2024-09-16T08:16:50.031 INFO:teuthology.orchestra.run.smithi028.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 12, 2024-09-16T08:16:50.031 INFO:teuthology.orchestra.run.smithi028.stdout: "ceph version 19.3.0-4917-g8293d73f (8293d73f8690540e843a81caec373f9cc29cf705) squid (dev)": 2 2024-09-16T08:16:50.031 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-09-16T08:16:50.031 INFO:teuthology.orchestra.run.smithi028.stdout:} 2024-09-16T08:16:50.501 INFO:teuthology.orchestra.run.smithi028.stdout:{ 2024-09-16T08:16:50.501 INFO:teuthology.orchestra.run.smithi028.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:f7ad55f9ccd17ba66aadb19b4d291e67b60ea24e5d8a0c0e92941fd59c81e4ce", 2024-09-16T08:16:50.502 INFO:teuthology.orchestra.run.smithi028.stdout: "in_progress": true, 2024-09-16T08:16:50.502 INFO:teuthology.orchestra.run.smithi028.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-16T08:16:50.502 INFO:teuthology.orchestra.run.smithi028.stdout: "services_complete": [ 2024-09-16T08:16:50.502 INFO:teuthology.orchestra.run.smithi028.stdout: "mgr" 2024-09-16T08:16:50.502 INFO:teuthology.orchestra.run.smithi028.stdout: ], 2024-09-16T08:16:50.502 INFO:teuthology.orchestra.run.smithi028.stdout: "progress": "2/23 daemons upgraded", 2024-09-16T08:16:50.502 INFO:teuthology.orchestra.run.smithi028.stdout: "message": "Currently upgrading mgr daemons", 2024-09-16T08:16:50.502 INFO:teuthology.orchestra.run.smithi028.stdout: "is_paused": false 2024-09-16T08:16:50.502 INFO:teuthology.orchestra.run.smithi028.stdout:} 2024-09-16T08:16:50.760 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:16:50 smithi028 ceph-mon[23744]: pgmap v55: 97 pgs: 97 active+clean; 332 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 6.9 KiB/s rd, 4.5 MiB/s wr, 457 op/s 2024-09-16T08:16:50.760 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:16:50 smithi028 ceph-mon[23744]: from='client.24561 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T08:16:50.760 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:16:50 smithi028 ceph-mon[23744]: from='client.24565 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T08:16:50.760 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:16:50 smithi028 ceph-mon[23744]: from='mgr.24459 ' entity='mgr.smithi144.utzszj' 2024-09-16T08:16:50.760 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:16:50 smithi028 ceph-mon[23744]: from='mgr.24459 ' entity='mgr.smithi144.utzszj' 2024-09-16T08:16:50.760 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:16:50 smithi028 ceph-mon[23744]: from='mgr.24459 172.21.15.144:0/4176577021' entity='mgr.smithi144.utzszj' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T08:16:50.760 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:16:50 smithi028 ceph-mon[23744]: from='mgr.24459 172.21.15.144:0/4176577021' entity='mgr.smithi144.utzszj' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-16T08:16:50.760 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:16:50 smithi028 ceph-mon[23744]: from='mgr.24459 ' entity='mgr.smithi144.utzszj' 2024-09-16T08:16:50.761 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:16:50 smithi028 ceph-mon[23744]: from='mgr.24459 ' entity='mgr.smithi144.utzszj' 2024-09-16T08:16:50.761 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:16:50 smithi028 ceph-mon[23744]: from='mgr.24459 172.21.15.144:0/4176577021' entity='mgr.smithi144.utzszj' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-16T08:16:50.761 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:16:50 smithi028 ceph-mon[23744]: from='mgr.24459 172.21.15.144:0/4176577021' entity='mgr.smithi144.utzszj' cmd=[{"prefix": "mgr fail", "who": "smithi144.utzszj"}]: dispatch 2024-09-16T08:16:50.761 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:16:50 smithi028 ceph-mon[23744]: from='mgr.24459 ' entity='mgr.smithi144.utzszj' cmd=[{"prefix": "mgr fail", "who": "smithi144.utzszj"}]: dispatch 2024-09-16T08:16:50.761 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:16:50 smithi028 ceph-mon[23744]: osdmap e71: 8 total, 8 up, 8 in 2024-09-16T08:16:50.761 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:16:50 smithi028 ceph-mon[23744]: from='client.? 172.21.15.28:0/4229427829' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-16T08:16:50.761 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:16:50 smithi028 ceph-mon[23744]: Standby manager daemon smithi144.utzszj started 2024-09-16T08:16:50.761 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:16:50 smithi028 ceph-mon[23744]: from='mgr.24459 172.21.15.144:0/4176577021' entity='mgr.smithi144.utzszj' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-16T08:16:50.831 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:16:50 smithi144 ceph-mon[25668]: pgmap v55: 97 pgs: 97 active+clean; 332 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 6.9 KiB/s rd, 4.5 MiB/s wr, 457 op/s 2024-09-16T08:16:50.832 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:16:50 smithi144 ceph-mon[25668]: from='client.24561 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T08:16:50.832 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:16:50 smithi144 ceph-mon[25668]: from='client.24565 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T08:16:50.832 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:16:50 smithi144 ceph-mon[25668]: from='mgr.24459 ' entity='mgr.smithi144.utzszj' 2024-09-16T08:16:50.832 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:16:50 smithi144 ceph-mon[25668]: from='mgr.24459 ' entity='mgr.smithi144.utzszj' 2024-09-16T08:16:50.832 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:16:50 smithi144 ceph-mon[25668]: from='mgr.24459 172.21.15.144:0/4176577021' entity='mgr.smithi144.utzszj' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T08:16:50.832 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:16:50 smithi144 ceph-mon[25668]: from='mgr.24459 172.21.15.144:0/4176577021' entity='mgr.smithi144.utzszj' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-16T08:16:50.832 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:16:50 smithi144 ceph-mon[25668]: from='mgr.24459 ' entity='mgr.smithi144.utzszj' 2024-09-16T08:16:50.832 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:16:50 smithi144 ceph-mon[25668]: from='mgr.24459 ' entity='mgr.smithi144.utzszj' 2024-09-16T08:16:50.832 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:16:50 smithi144 ceph-mon[25668]: from='mgr.24459 172.21.15.144:0/4176577021' entity='mgr.smithi144.utzszj' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-16T08:16:50.832 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:16:50 smithi144 ceph-mon[25668]: from='mgr.24459 172.21.15.144:0/4176577021' entity='mgr.smithi144.utzszj' cmd=[{"prefix": "mgr fail", "who": "smithi144.utzszj"}]: dispatch 2024-09-16T08:16:50.832 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:16:50 smithi144 ceph-mon[25668]: from='mgr.24459 ' entity='mgr.smithi144.utzszj' cmd=[{"prefix": "mgr fail", "who": "smithi144.utzszj"}]: dispatch 2024-09-16T08:16:50.832 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:16:50 smithi144 ceph-mon[25668]: osdmap e71: 8 total, 8 up, 8 in 2024-09-16T08:16:50.833 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:16:50 smithi144 ceph-mon[25668]: from='client.? 172.21.15.28:0/4229427829' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-16T08:16:50.833 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:16:50 smithi144 ceph-mon[25668]: Standby manager daemon smithi144.utzszj started 2024-09-16T08:16:50.833 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:16:50 smithi144 ceph-mon[25668]: from='mgr.24459 172.21.15.144:0/4176577021' entity='mgr.smithi144.utzszj' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-16T08:16:51.024 INFO:teuthology.orchestra.run.smithi028.stdout: 5 116744 11.83 MB/sec execute 109 sec latency 75.985 ms 2024-09-16T08:16:51.221 INFO:teuthology.orchestra.run.smithi028.stdout:HEALTH_OK 2024-09-16T08:16:51.777 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:16:51 smithi028 ceph-mon[23744]: from='mgr.24459 ' entity='mgr.smithi144.utzszj' cmd='[{"prefix": "mgr fail", "who": "smithi144.utzszj"}]': finished 2024-09-16T08:16:51.777 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:16:51 smithi028 ceph-mon[23744]: mgrmap e30: smithi028.azvyzy(active, starting, since 0.7063s), standbys: smithi144.utzszj 2024-09-16T08:16:51.777 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:16:51 smithi028 ceph-mon[23744]: from='mgr.14768 172.21.15.28:0/1901954942' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "mon metadata", "id": "smithi028"}]: dispatch 2024-09-16T08:16:51.777 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:16:51 smithi028 ceph-mon[23744]: from='mgr.14768 172.21.15.28:0/1901954942' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "mon metadata", "id": "smithi144"}]: dispatch 2024-09-16T08:16:51.778 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:16:51 smithi028 ceph-mon[23744]: from='mgr.14768 172.21.15.28:0/1901954942' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi028.srtgev"}]: dispatch 2024-09-16T08:16:51.778 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:16:51 smithi028 ceph-mon[23744]: from='mgr.14768 172.21.15.28:0/1901954942' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi144.oozgsi"}]: dispatch 2024-09-16T08:16:51.778 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:16:51 smithi028 ceph-mon[23744]: from='mgr.14768 172.21.15.28:0/1901954942' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "mgr metadata", "who": "smithi028.azvyzy", "id": "smithi028.azvyzy"}]: dispatch 2024-09-16T08:16:51.778 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:16:51 smithi028 ceph-mon[23744]: from='mgr.14768 172.21.15.28:0/1901954942' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "mgr metadata", "who": "smithi144.utzszj", "id": "smithi144.utzszj"}]: dispatch 2024-09-16T08:16:51.778 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:16:51 smithi028 ceph-mon[23744]: from='mgr.14768 172.21.15.28:0/1901954942' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-16T08:16:51.778 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:16:51 smithi028 ceph-mon[23744]: from='mgr.14768 172.21.15.28:0/1901954942' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-16T08:16:51.778 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:16:51 smithi028 ceph-mon[23744]: from='mgr.14768 172.21.15.28:0/1901954942' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-16T08:16:51.778 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:16:51 smithi028 ceph-mon[23744]: from='mgr.14768 172.21.15.28:0/1901954942' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-16T08:16:51.778 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:16:51 smithi028 ceph-mon[23744]: from='mgr.14768 172.21.15.28:0/1901954942' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-16T08:16:51.778 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:16:51 smithi028 ceph-mon[23744]: from='mgr.14768 172.21.15.28:0/1901954942' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-16T08:16:51.779 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:16:51 smithi028 ceph-mon[23744]: from='mgr.14768 172.21.15.28:0/1901954942' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-16T08:16:51.779 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:16:51 smithi028 ceph-mon[23744]: from='mgr.14768 172.21.15.28:0/1901954942' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-16T08:16:51.779 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:16:51 smithi028 ceph-mon[23744]: from='mgr.14768 172.21.15.28:0/1901954942' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-16T08:16:51.779 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:16:51 smithi028 ceph-mon[23744]: from='mgr.14768 172.21.15.28:0/1901954942' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-16T08:16:51.779 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:16:51 smithi028 ceph-mon[23744]: from='mgr.14768 172.21.15.28:0/1901954942' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-16T08:16:51.779 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:16:51 smithi028 ceph-mon[23744]: Manager daemon smithi028.azvyzy is now available 2024-09-16T08:16:51.779 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:16:51 smithi028 ceph-mon[23744]: from='client.? 172.21.15.28:0/746539017' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-16T08:16:52.024 INFO:teuthology.orchestra.run.smithi028.stdout: 5 117079 11.84 MB/sec execute 110 sec latency 48.924 ms 2024-09-16T08:16:52.082 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:16:51 smithi144 ceph-mon[25668]: from='mgr.24459 ' entity='mgr.smithi144.utzszj' cmd='[{"prefix": "mgr fail", "who": "smithi144.utzszj"}]': finished 2024-09-16T08:16:52.082 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:16:51 smithi144 ceph-mon[25668]: mgrmap e30: smithi028.azvyzy(active, starting, since 0.7063s), standbys: smithi144.utzszj 2024-09-16T08:16:52.082 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:16:51 smithi144 ceph-mon[25668]: from='mgr.14768 172.21.15.28:0/1901954942' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "mon metadata", "id": "smithi028"}]: dispatch 2024-09-16T08:16:52.082 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:16:51 smithi144 ceph-mon[25668]: from='mgr.14768 172.21.15.28:0/1901954942' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "mon metadata", "id": "smithi144"}]: dispatch 2024-09-16T08:16:52.082 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:16:51 smithi144 ceph-mon[25668]: from='mgr.14768 172.21.15.28:0/1901954942' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi028.srtgev"}]: dispatch 2024-09-16T08:16:52.082 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:16:51 smithi144 ceph-mon[25668]: from='mgr.14768 172.21.15.28:0/1901954942' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi144.oozgsi"}]: dispatch 2024-09-16T08:16:52.082 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:16:51 smithi144 ceph-mon[25668]: from='mgr.14768 172.21.15.28:0/1901954942' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "mgr metadata", "who": "smithi028.azvyzy", "id": "smithi028.azvyzy"}]: dispatch 2024-09-16T08:16:52.082 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:16:51 smithi144 ceph-mon[25668]: from='mgr.14768 172.21.15.28:0/1901954942' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "mgr metadata", "who": "smithi144.utzszj", "id": "smithi144.utzszj"}]: dispatch 2024-09-16T08:16:52.083 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:16:51 smithi144 ceph-mon[25668]: from='mgr.14768 172.21.15.28:0/1901954942' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-16T08:16:52.083 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:16:51 smithi144 ceph-mon[25668]: from='mgr.14768 172.21.15.28:0/1901954942' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-16T08:16:52.083 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:16:51 smithi144 ceph-mon[25668]: from='mgr.14768 172.21.15.28:0/1901954942' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-16T08:16:52.083 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:16:51 smithi144 ceph-mon[25668]: from='mgr.14768 172.21.15.28:0/1901954942' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-16T08:16:52.083 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:16:51 smithi144 ceph-mon[25668]: from='mgr.14768 172.21.15.28:0/1901954942' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-16T08:16:52.083 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:16:51 smithi144 ceph-mon[25668]: from='mgr.14768 172.21.15.28:0/1901954942' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-16T08:16:52.083 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:16:51 smithi144 ceph-mon[25668]: from='mgr.14768 172.21.15.28:0/1901954942' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-16T08:16:52.083 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:16:51 smithi144 ceph-mon[25668]: from='mgr.14768 172.21.15.28:0/1901954942' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-16T08:16:52.083 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:16:51 smithi144 ceph-mon[25668]: from='mgr.14768 172.21.15.28:0/1901954942' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-16T08:16:52.083 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:16:51 smithi144 ceph-mon[25668]: from='mgr.14768 172.21.15.28:0/1901954942' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-16T08:16:52.083 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:16:51 smithi144 ceph-mon[25668]: from='mgr.14768 172.21.15.28:0/1901954942' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-16T08:16:52.083 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:16:51 smithi144 ceph-mon[25668]: Manager daemon smithi028.azvyzy is now available 2024-09-16T08:16:52.083 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:16:51 smithi144 ceph-mon[25668]: from='client.? 172.21.15.28:0/746539017' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-16T08:16:52.804 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:16:52 smithi028 ceph-mon[23744]: from='mgr.14768 172.21.15.28:0/1901954942' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-16T08:16:52.805 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:16:52 smithi028 ceph-mon[23744]: from='mgr.14768 172.21.15.28:0/1901954942' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-16T08:16:52.805 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:16:52 smithi028 ceph-mon[23744]: from='mgr.14768 172.21.15.28:0/1901954942' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-16T08:16:52.805 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:16:52 smithi028 ceph-mon[23744]: from='mgr.14768 172.21.15.28:0/1901954942' entity='mgr.smithi028.azvyzy' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi028.azvyzy/mirror_snapshot_schedule"}]: dispatch 2024-09-16T08:16:52.805 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:16:52 smithi028 ceph-mon[23744]: from='mgr.14768 ' entity='mgr.smithi028.azvyzy' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi028.azvyzy/mirror_snapshot_schedule"}]: dispatch 2024-09-16T08:16:52.805 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:16:52 smithi028 ceph-mon[23744]: mgrmap e31: smithi028.azvyzy(active, since 1.74334s), standbys: smithi144.utzszj 2024-09-16T08:16:52.805 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:16:52 smithi028 ceph-mon[23744]: from='mgr.14768 172.21.15.28:0/1901954942' entity='mgr.smithi028.azvyzy' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi028.azvyzy/trash_purge_schedule"}]: dispatch 2024-09-16T08:16:52.805 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:16:52 smithi028 ceph-mon[23744]: from='mgr.14768 ' entity='mgr.smithi028.azvyzy' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi028.azvyzy/trash_purge_schedule"}]: dispatch 2024-09-16T08:16:52.805 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:16:52 smithi028 ceph-mon[23744]: pgmap v3: 97 pgs: 97 active+clean; 334 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2024-09-16T08:16:52.805 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:16:52 smithi028 ceph-mon[23744]: from='mon.? -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-16T08:16:52.896 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:16:52 smithi144 ceph-mon[25668]: from='mgr.14768 172.21.15.28:0/1901954942' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-16T08:16:52.896 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:16:52 smithi144 ceph-mon[25668]: from='mgr.14768 172.21.15.28:0/1901954942' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-16T08:16:52.896 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:16:52 smithi144 ceph-mon[25668]: from='mgr.14768 172.21.15.28:0/1901954942' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-16T08:16:52.896 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:16:52 smithi144 ceph-mon[25668]: from='mgr.14768 172.21.15.28:0/1901954942' entity='mgr.smithi028.azvyzy' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi028.azvyzy/mirror_snapshot_schedule"}]: dispatch 2024-09-16T08:16:52.897 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:16:52 smithi144 ceph-mon[25668]: from='mgr.14768 ' entity='mgr.smithi028.azvyzy' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi028.azvyzy/mirror_snapshot_schedule"}]: dispatch 2024-09-16T08:16:52.897 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:16:52 smithi144 ceph-mon[25668]: mgrmap e31: smithi028.azvyzy(active, since 1.74334s), standbys: smithi144.utzszj 2024-09-16T08:16:52.897 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:16:52 smithi144 ceph-mon[25668]: from='mgr.14768 172.21.15.28:0/1901954942' entity='mgr.smithi028.azvyzy' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi028.azvyzy/trash_purge_schedule"}]: dispatch 2024-09-16T08:16:52.897 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:16:52 smithi144 ceph-mon[25668]: from='mgr.14768 ' entity='mgr.smithi028.azvyzy' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi028.azvyzy/trash_purge_schedule"}]: dispatch 2024-09-16T08:16:52.897 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:16:52 smithi144 ceph-mon[25668]: pgmap v3: 97 pgs: 97 active+clean; 334 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2024-09-16T08:16:52.897 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:16:52 smithi144 ceph-mon[25668]: from='mon.? -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-16T08:16:53.024 INFO:teuthology.orchestra.run.smithi028.stdout: 5 117500 11.84 MB/sec execute 111 sec latency 33.441 ms 2024-09-16T08:16:53.819 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:16:53 smithi028 ceph-mon[23744]: [16/Sep/2024:08:16:53] ENGINE Bus STARTING 2024-09-16T08:16:53.820 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:16:53 smithi028 ceph-mon[23744]: [16/Sep/2024:08:16:53] ENGINE Serving on http://172.21.15.28:8765 2024-09-16T08:16:53.820 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:16:53 smithi028 ceph-mon[23744]: [16/Sep/2024:08:16:53] ENGINE Serving on https://172.21.15.28:7150 2024-09-16T08:16:53.820 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:16:53 smithi028 ceph-mon[23744]: [16/Sep/2024:08:16:53] ENGINE Bus STARTED 2024-09-16T08:16:53.997 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:16:53 smithi144 ceph-mon[25668]: [16/Sep/2024:08:16:53] ENGINE Bus STARTING 2024-09-16T08:16:53.997 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:16:53 smithi144 ceph-mon[25668]: [16/Sep/2024:08:16:53] ENGINE Serving on http://172.21.15.28:8765 2024-09-16T08:16:53.997 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:16:53 smithi144 ceph-mon[25668]: [16/Sep/2024:08:16:53] ENGINE Serving on https://172.21.15.28:7150 2024-09-16T08:16:53.997 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:16:53 smithi144 ceph-mon[25668]: [16/Sep/2024:08:16:53] ENGINE Bus STARTED 2024-09-16T08:16:54.024 INFO:teuthology.orchestra.run.smithi028.stdout: 5 117789 11.78 MB/sec execute 112 sec latency 51.457 ms 2024-09-16T08:16:54.824 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:16:54 smithi028 ceph-mon[23744]: [16/Sep/2024:08:16:53] ENGINE Error in HTTPServer.serve 2024-09-16T08:16:54.824 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout: Traceback (most recent call last): 2024-09-16T08:16:54.824 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-16T08:16:54.824 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout: self._connections.run(self.expiration_interval) 2024-09-16T08:16:54.824 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-16T08:16:54.824 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout: self._run(expiration_interval) 2024-09-16T08:16:54.824 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-16T08:16:54.824 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-16T08:16:54.824 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-16T08:16:54.824 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-16T08:16:54.825 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-16T08:16:54.825 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout: s = self.context.wrap_socket( 2024-09-16T08:16:54.825 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-16T08:16:54.825 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout: return self.sslsocket_class._create( 2024-09-16T08:16:54.825 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-16T08:16:54.825 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout: self.do_handshake() 2024-09-16T08:16:54.825 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-16T08:16:54.825 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout: self._sslobj.do_handshake() 2024-09-16T08:16:54.825 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-16T08:16:54.825 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:16:54 smithi028 ceph-mon[23744]: mgrmap e32: smithi028.azvyzy(active, since 3s), standbys: smithi144.utzszj 2024-09-16T08:16:54.831 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:16:54 smithi144 ceph-mon[25668]: [16/Sep/2024:08:16:53] ENGINE Error in HTTPServer.serve 2024-09-16T08:16:54.831 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: Traceback (most recent call last): 2024-09-16T08:16:54.831 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-16T08:16:54.831 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: self._connections.run(self.expiration_interval) 2024-09-16T08:16:54.831 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-16T08:16:54.831 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: self._run(expiration_interval) 2024-09-16T08:16:54.832 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-16T08:16:54.832 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-16T08:16:54.832 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-16T08:16:54.832 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-16T08:16:54.832 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-16T08:16:54.832 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: s = self.context.wrap_socket( 2024-09-16T08:16:54.832 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-16T08:16:54.832 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: return self.sslsocket_class._create( 2024-09-16T08:16:54.832 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-16T08:16:54.832 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: self.do_handshake() 2024-09-16T08:16:54.832 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-16T08:16:54.832 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: self._sslobj.do_handshake() 2024-09-16T08:16:54.832 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-16T08:16:54.833 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:16:54 smithi144 ceph-mon[25668]: mgrmap e32: smithi028.azvyzy(active, since 3s), standbys: smithi144.utzszj 2024-09-16T08:16:55.024 INFO:teuthology.orchestra.run.smithi028.stdout: 5 118137 11.75 MB/sec execute 113 sec latency 37.486 ms 2024-09-16T08:16:56.024 INFO:teuthology.orchestra.run.smithi028.stdout: 5 118439 11.67 MB/sec execute 114 sec latency 58.664 ms 2024-09-16T08:16:56.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:16:56 smithi144 ceph-mon[25668]: pgmap v5: 97 pgs: 97 active+clean; 334 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2024-09-16T08:16:56.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:16:56 smithi144 ceph-mon[25668]: mgrmap e33: smithi028.azvyzy(active, since 5s), standbys: smithi144.utzszj 2024-09-16T08:16:56.359 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:16:56 smithi028 ceph-mon[23744]: pgmap v5: 97 pgs: 97 active+clean; 334 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2024-09-16T08:16:56.359 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:16:56 smithi028 ceph-mon[23744]: mgrmap e33: smithi028.azvyzy(active, since 5s), standbys: smithi144.utzszj 2024-09-16T08:16:57.024 INFO:teuthology.orchestra.run.smithi028.stdout: 5 118749 11.63 MB/sec execute 115 sec latency 50.755 ms 2024-09-16T08:16:57.387 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:16:57 smithi028 ceph-mon[23744]: from='mgr.14768 ' entity='mgr.smithi028.azvyzy' 2024-09-16T08:16:57.387 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:16:57 smithi028 ceph-mon[23744]: from='mgr.14768 ' entity='mgr.smithi028.azvyzy' 2024-09-16T08:16:57.387 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:16:57 smithi028 ceph-mon[23744]: Standby manager daemon smithi144.utzszj restarted 2024-09-16T08:16:57.388 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:16:57 smithi028 ceph-mon[23744]: Standby manager daemon smithi144.utzszj started 2024-09-16T08:16:57.388 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:16:57 smithi028 ceph-mon[23744]: from='mgr.? 172.21.15.144:0/586563211' entity='mgr.smithi144.utzszj' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi144.utzszj/crt"}]: dispatch 2024-09-16T08:16:57.388 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:16:57 smithi028 ceph-mon[23744]: from='mgr.? 172.21.15.144:0/586563211' entity='mgr.smithi144.utzszj' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-16T08:16:57.388 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:16:57 smithi028 ceph-mon[23744]: from='mgr.? 172.21.15.144:0/586563211' entity='mgr.smithi144.utzszj' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi144.utzszj/key"}]: dispatch 2024-09-16T08:16:57.388 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:16:57 smithi028 ceph-mon[23744]: from='mgr.? 172.21.15.144:0/586563211' entity='mgr.smithi144.utzszj' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-16T08:16:57.388 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:16:57 smithi028 ceph-mon[23744]: from='mgr.14768 ' entity='mgr.smithi028.azvyzy' 2024-09-16T08:16:57.388 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:16:57 smithi028 ceph-mon[23744]: from='mgr.14768 ' entity='mgr.smithi028.azvyzy' 2024-09-16T08:16:57.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:16:57 smithi144 ceph-mon[25668]: from='mgr.14768 ' entity='mgr.smithi028.azvyzy' 2024-09-16T08:16:57.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:16:57 smithi144 ceph-mon[25668]: from='mgr.14768 ' entity='mgr.smithi028.azvyzy' 2024-09-16T08:16:57.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:16:57 smithi144 ceph-mon[25668]: Standby manager daemon smithi144.utzszj restarted 2024-09-16T08:16:57.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:16:57 smithi144 ceph-mon[25668]: Standby manager daemon smithi144.utzszj started 2024-09-16T08:16:57.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:16:57 smithi144 ceph-mon[25668]: from='mgr.? 172.21.15.144:0/586563211' entity='mgr.smithi144.utzszj' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi144.utzszj/crt"}]: dispatch 2024-09-16T08:16:57.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:16:57 smithi144 ceph-mon[25668]: from='mgr.? 172.21.15.144:0/586563211' entity='mgr.smithi144.utzszj' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-16T08:16:57.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:16:57 smithi144 ceph-mon[25668]: from='mgr.? 172.21.15.144:0/586563211' entity='mgr.smithi144.utzszj' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi144.utzszj/key"}]: dispatch 2024-09-16T08:16:57.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:16:57 smithi144 ceph-mon[25668]: from='mgr.? 172.21.15.144:0/586563211' entity='mgr.smithi144.utzszj' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-16T08:16:57.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:16:57 smithi144 ceph-mon[25668]: from='mgr.14768 ' entity='mgr.smithi028.azvyzy' 2024-09-16T08:16:57.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:16:57 smithi144 ceph-mon[25668]: from='mgr.14768 ' entity='mgr.smithi028.azvyzy' 2024-09-16T08:16:58.024 INFO:teuthology.orchestra.run.smithi028.stdout: 5 119033 11.59 MB/sec execute 116 sec latency 59.601 ms 2024-09-16T08:16:58.413 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:16:58 smithi028 ceph-mon[23744]: pgmap v6: 97 pgs: 97 active+clean; 338 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 9.8 KiB/s rd, 1.5 MiB/s wr, 244 op/s 2024-09-16T08:16:58.414 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:16:58 smithi028 ceph-mon[23744]: mgrmap e34: smithi028.azvyzy(active, since 7s), standbys: smithi144.utzszj 2024-09-16T08:16:58.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:16:58 smithi144 ceph-mon[25668]: pgmap v6: 97 pgs: 97 active+clean; 338 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 9.8 KiB/s rd, 1.5 MiB/s wr, 244 op/s 2024-09-16T08:16:58.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:16:58 smithi144 ceph-mon[25668]: mgrmap e34: smithi028.azvyzy(active, since 7s), standbys: smithi144.utzszj 2024-09-16T08:16:59.025 INFO:teuthology.orchestra.run.smithi028.stdout: 5 119440 11.60 MB/sec execute 117 sec latency 53.330 ms 2024-09-16T08:17:00.025 INFO:teuthology.orchestra.run.smithi028.stdout: 5 119815 11.56 MB/sec execute 118 sec latency 58.431 ms 2024-09-16T08:17:00.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:00 smithi028 ceph-mon[23744]: pgmap v7: 97 pgs: 97 active+clean; 328 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 33 KiB/s rd, 3.4 MiB/s wr, 380 op/s 2024-09-16T08:17:00.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:17:00 smithi144 ceph-mon[25668]: pgmap v7: 97 pgs: 97 active+clean; 328 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 33 KiB/s rd, 3.4 MiB/s wr, 380 op/s 2024-09-16T08:17:01.025 INFO:teuthology.orchestra.run.smithi028.stdout: 5 120086 11.58 MB/sec execute 119 sec latency 61.672 ms 2024-09-16T08:17:02.018 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:02 smithi028 ceph-mon[23744]: pgmap v8: 97 pgs: 97 active+clean; 328 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 25 KiB/s rd, 2.6 MiB/s wr, 294 op/s 2024-09-16T08:17:02.018 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:02 smithi028 ceph-mon[23744]: from='mgr.14768 ' entity='mgr.smithi028.azvyzy' 2024-09-16T08:17:02.018 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:02 smithi028 ceph-mon[23744]: from='mgr.14768 ' entity='mgr.smithi028.azvyzy' 2024-09-16T08:17:02.018 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:02 smithi028 ceph-mon[23744]: from='mgr.14768 ' entity='mgr.smithi028.azvyzy' 2024-09-16T08:17:02.018 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:02 smithi028 ceph-mon[23744]: from='mgr.14768 ' entity='mgr.smithi028.azvyzy' 2024-09-16T08:17:02.024 INFO:teuthology.orchestra.run.smithi028.stdout: 5 120354 11.57 MB/sec execute 120 sec latency 62.981 ms 2024-09-16T08:17:02.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:17:02 smithi144 ceph-mon[25668]: pgmap v8: 97 pgs: 97 active+clean; 328 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 25 KiB/s rd, 2.6 MiB/s wr, 294 op/s 2024-09-16T08:17:02.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:17:02 smithi144 ceph-mon[25668]: from='mgr.14768 ' entity='mgr.smithi028.azvyzy' 2024-09-16T08:17:02.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:17:02 smithi144 ceph-mon[25668]: from='mgr.14768 ' entity='mgr.smithi028.azvyzy' 2024-09-16T08:17:02.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:17:02 smithi144 ceph-mon[25668]: from='mgr.14768 ' entity='mgr.smithi028.azvyzy' 2024-09-16T08:17:02.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:17:02 smithi144 ceph-mon[25668]: from='mgr.14768 ' entity='mgr.smithi028.azvyzy' 2024-09-16T08:17:03.025 INFO:teuthology.orchestra.run.smithi028.stdout: 5 120714 11.57 MB/sec execute 121 sec latency 53.137 ms 2024-09-16T08:17:04.025 INFO:teuthology.orchestra.run.smithi028.stdout: 5 121096 11.57 MB/sec execute 122 sec latency 42.903 ms 2024-09-16T08:17:04.046 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:04 smithi028 ceph-mon[23744]: pgmap v9: 97 pgs: 97 active+clean; 328 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 23 KiB/s rd, 4.4 MiB/s wr, 450 op/s 2024-09-16T08:17:04.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:17:04 smithi144 ceph-mon[25668]: pgmap v9: 97 pgs: 97 active+clean; 328 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 23 KiB/s rd, 4.4 MiB/s wr, 450 op/s 2024-09-16T08:17:05.025 INFO:teuthology.orchestra.run.smithi028.stdout: 5 121359 11.51 MB/sec execute 123 sec latency 55.601 ms 2024-09-16T08:17:06.025 INFO:teuthology.orchestra.run.smithi028.stdout: 5 121723 11.48 MB/sec execute 124 sec latency 49.848 ms 2024-09-16T08:17:06.309 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:06 smithi028 ceph-mon[23744]: pgmap v10: 97 pgs: 97 active+clean; 328 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 21 KiB/s rd, 4.0 MiB/s wr, 409 op/s 2024-09-16T08:17:06.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:17:06 smithi144 ceph-mon[25668]: pgmap v10: 97 pgs: 97 active+clean; 328 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 21 KiB/s rd, 4.0 MiB/s wr, 409 op/s 2024-09-16T08:17:07.025 INFO:teuthology.orchestra.run.smithi028.stdout: 5 122129 11.42 MB/sec execute 125 sec latency 120.506 ms 2024-09-16T08:17:07.342 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:07 smithi028 ceph-mon[23744]: from='mgr.14768 172.21.15.28:0/1901954942' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-16T08:17:07.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:17:07 smithi144 ceph-mon[25668]: from='mgr.14768 172.21.15.28:0/1901954942' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-16T08:17:08.025 INFO:teuthology.orchestra.run.smithi028.stdout: 5 122439 11.39 MB/sec execute 126 sec latency 63.636 ms 2024-09-16T08:17:08.355 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:08 smithi028 ceph-mon[23744]: pgmap v11: 97 pgs: 97 active+clean; 330 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 24 KiB/s rd, 5.1 MiB/s wr, 521 op/s 2024-09-16T08:17:08.355 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:08 smithi028 ceph-mon[23744]: from='mgr.14768 ' entity='mgr.smithi028.azvyzy' 2024-09-16T08:17:08.355 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:08 smithi028 ceph-mon[23744]: from='mgr.14768 ' entity='mgr.smithi028.azvyzy' 2024-09-16T08:17:08.355 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:08 smithi028 ceph-mon[23744]: from='mgr.14768 172.21.15.28:0/1901954942' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T08:17:08.355 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:08 smithi028 ceph-mon[23744]: from='mgr.14768 172.21.15.28:0/1901954942' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-16T08:17:08.355 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:08 smithi028 ceph-mon[23744]: Updating smithi028:/etc/ceph/ceph.conf 2024-09-16T08:17:08.355 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:08 smithi028 ceph-mon[23744]: Updating smithi144:/etc/ceph/ceph.conf 2024-09-16T08:17:08.356 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:08 smithi028 ceph-mon[23744]: Updating smithi144:/var/lib/ceph/5622b2fe-7402-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-16T08:17:08.356 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:08 smithi028 ceph-mon[23744]: Updating smithi028:/var/lib/ceph/5622b2fe-7402-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-16T08:17:08.356 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:08 smithi028 ceph-mon[23744]: Updating smithi028:/etc/ceph/ceph.client.admin.keyring 2024-09-16T08:17:08.356 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:08 smithi028 ceph-mon[23744]: from='mgr.14768 ' entity='mgr.smithi028.azvyzy' 2024-09-16T08:17:08.356 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:08 smithi028 ceph-mon[23744]: from='mgr.14768 ' entity='mgr.smithi028.azvyzy' 2024-09-16T08:17:08.356 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:08 smithi028 ceph-mon[23744]: from='mgr.14768 ' entity='mgr.smithi028.azvyzy' 2024-09-16T08:17:08.356 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:08 smithi028 ceph-mon[23744]: from='mgr.14768 ' entity='mgr.smithi028.azvyzy' 2024-09-16T08:17:08.356 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:08 smithi028 ceph-mon[23744]: from='mgr.14768 ' entity='mgr.smithi028.azvyzy' 2024-09-16T08:17:08.356 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:08 smithi028 ceph-mon[23744]: from='mgr.14768 ' entity='mgr.smithi028.azvyzy' 2024-09-16T08:17:08.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:17:08 smithi144 ceph-mon[25668]: pgmap v11: 97 pgs: 97 active+clean; 330 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 24 KiB/s rd, 5.1 MiB/s wr, 521 op/s 2024-09-16T08:17:08.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:17:08 smithi144 ceph-mon[25668]: from='mgr.14768 ' entity='mgr.smithi028.azvyzy' 2024-09-16T08:17:08.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:17:08 smithi144 ceph-mon[25668]: from='mgr.14768 ' entity='mgr.smithi028.azvyzy' 2024-09-16T08:17:08.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:17:08 smithi144 ceph-mon[25668]: from='mgr.14768 172.21.15.28:0/1901954942' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T08:17:08.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:17:08 smithi144 ceph-mon[25668]: from='mgr.14768 172.21.15.28:0/1901954942' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-16T08:17:08.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:17:08 smithi144 ceph-mon[25668]: Updating smithi028:/etc/ceph/ceph.conf 2024-09-16T08:17:08.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:17:08 smithi144 ceph-mon[25668]: Updating smithi144:/etc/ceph/ceph.conf 2024-09-16T08:17:08.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:17:08 smithi144 ceph-mon[25668]: Updating smithi144:/var/lib/ceph/5622b2fe-7402-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-16T08:17:08.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:17:08 smithi144 ceph-mon[25668]: Updating smithi028:/var/lib/ceph/5622b2fe-7402-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-16T08:17:08.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:17:08 smithi144 ceph-mon[25668]: Updating smithi028:/etc/ceph/ceph.client.admin.keyring 2024-09-16T08:17:08.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:17:08 smithi144 ceph-mon[25668]: from='mgr.14768 ' entity='mgr.smithi028.azvyzy' 2024-09-16T08:17:08.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:17:08 smithi144 ceph-mon[25668]: from='mgr.14768 ' entity='mgr.smithi028.azvyzy' 2024-09-16T08:17:08.583 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:17:08 smithi144 ceph-mon[25668]: from='mgr.14768 ' entity='mgr.smithi028.azvyzy' 2024-09-16T08:17:08.583 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:17:08 smithi144 ceph-mon[25668]: from='mgr.14768 ' entity='mgr.smithi028.azvyzy' 2024-09-16T08:17:08.583 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:17:08 smithi144 ceph-mon[25668]: from='mgr.14768 ' entity='mgr.smithi028.azvyzy' 2024-09-16T08:17:08.583 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:17:08 smithi144 ceph-mon[25668]: from='mgr.14768 ' entity='mgr.smithi028.azvyzy' 2024-09-16T08:17:09.025 INFO:teuthology.orchestra.run.smithi028.stdout: 5 122870 11.41 MB/sec execute 127 sec latency 51.964 ms 2024-09-16T08:17:09.247 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:09 smithi028 ceph-mon[23744]: Updating smithi144:/etc/ceph/ceph.client.admin.keyring 2024-09-16T08:17:09.248 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:09 smithi028 ceph-mon[23744]: Updating smithi028:/var/lib/ceph/5622b2fe-7402-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-16T08:17:09.248 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:09 smithi028 ceph-mon[23744]: Updating smithi144:/var/lib/ceph/5622b2fe-7402-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-16T08:17:09.248 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:09 smithi028 ceph-mon[23744]: Reconfiguring prometheus.smithi028 (dependencies changed)... 2024-09-16T08:17:09.248 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:09 smithi028 ceph-mon[23744]: Reconfiguring daemon prometheus.smithi028 on smithi028 2024-09-16T08:17:09.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:17:09 smithi144 ceph-mon[25668]: Updating smithi144:/etc/ceph/ceph.client.admin.keyring 2024-09-16T08:17:09.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:17:09 smithi144 ceph-mon[25668]: Updating smithi028:/var/lib/ceph/5622b2fe-7402-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-16T08:17:09.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:17:09 smithi144 ceph-mon[25668]: Updating smithi144:/var/lib/ceph/5622b2fe-7402-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-16T08:17:09.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:17:09 smithi144 ceph-mon[25668]: Reconfiguring prometheus.smithi028 (dependencies changed)... 2024-09-16T08:17:09.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:17:09 smithi144 ceph-mon[25668]: Reconfiguring daemon prometheus.smithi028 on smithi028 2024-09-16T08:17:10.025 INFO:teuthology.orchestra.run.smithi028.stdout: 5 123277 11.40 MB/sec execute 128 sec latency 45.644 ms 2024-09-16T08:17:10.265 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:10 smithi028 ceph-mon[23744]: pgmap v12: 97 pgs: 97 active+clean; 331 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 20 KiB/s rd, 5.5 MiB/s wr, 524 op/s 2024-09-16T08:17:10.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:17:10 smithi144 ceph-mon[25668]: pgmap v12: 97 pgs: 97 active+clean; 331 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 20 KiB/s rd, 5.5 MiB/s wr, 524 op/s 2024-09-16T08:17:11.025 INFO:teuthology.orchestra.run.smithi028.stdout: 5 123583 11.40 MB/sec execute 129 sec latency 57.595 ms 2024-09-16T08:17:12.025 INFO:teuthology.orchestra.run.smithi028.stdout: 5 123868 11.39 MB/sec execute 130 sec latency 69.256 ms 2024-09-16T08:17:12.323 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:12 smithi028 ceph-mon[23744]: pgmap v13: 97 pgs: 97 active+clean; 331 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 5.6 KiB/s rd, 4.1 MiB/s wr, 405 op/s 2024-09-16T08:17:12.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:17:12 smithi144 ceph-mon[25668]: pgmap v13: 97 pgs: 97 active+clean; 331 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 5.6 KiB/s rd, 4.1 MiB/s wr, 405 op/s 2024-09-16T08:17:13.025 INFO:teuthology.orchestra.run.smithi028.stdout: 5 124180 11.41 MB/sec execute 131 sec latency 64.660 ms 2024-09-16T08:17:14.025 INFO:teuthology.orchestra.run.smithi028.stdout: 5 124532 11.38 MB/sec execute 132 sec latency 39.950 ms 2024-09-16T08:17:14.377 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:14 smithi028 ceph-mon[23744]: pgmap v14: 97 pgs: 97 active+clean; 327 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 8.7 KiB/s rd, 6.3 MiB/s wr, 615 op/s 2024-09-16T08:17:14.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:17:14 smithi144 ceph-mon[25668]: pgmap v14: 97 pgs: 97 active+clean; 327 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 8.7 KiB/s rd, 6.3 MiB/s wr, 615 op/s 2024-09-16T08:17:15.025 INFO:teuthology.orchestra.run.smithi028.stdout: 5 124759 11.35 MB/sec execute 133 sec latency 41.102 ms 2024-09-16T08:17:16.034 INFO:teuthology.orchestra.run.smithi028.stdout: 5 125090 11.31 MB/sec execute 134 sec latency 46.099 ms 2024-09-16T08:17:16.420 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:16 smithi028 ceph-mon[23744]: pgmap v15: 97 pgs: 97 active+clean; 327 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 6.4 KiB/s rd, 4.3 MiB/s wr, 424 op/s 2024-09-16T08:17:16.420 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:16 smithi028 ceph-mon[23744]: from='mgr.14768 ' entity='mgr.smithi028.azvyzy' 2024-09-16T08:17:16.420 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:16 smithi028 ceph-mon[23744]: from='mgr.14768 ' entity='mgr.smithi028.azvyzy' 2024-09-16T08:17:16.420 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:16 smithi028 ceph-mon[23744]: from='mgr.14768 172.21.15.28:0/1901954942' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-16T08:17:16.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:17:16 smithi144 ceph-mon[25668]: pgmap v15: 97 pgs: 97 active+clean; 327 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 6.4 KiB/s rd, 4.3 MiB/s wr, 424 op/s 2024-09-16T08:17:16.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:17:16 smithi144 ceph-mon[25668]: from='mgr.14768 ' entity='mgr.smithi028.azvyzy' 2024-09-16T08:17:16.583 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:17:16 smithi144 ceph-mon[25668]: from='mgr.14768 ' entity='mgr.smithi028.azvyzy' 2024-09-16T08:17:16.583 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:17:16 smithi144 ceph-mon[25668]: from='mgr.14768 172.21.15.28:0/1901954942' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-16T08:17:17.034 INFO:teuthology.orchestra.run.smithi028.stdout: 5 125388 11.26 MB/sec execute 135 sec latency 53.894 ms 2024-09-16T08:17:17.433 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:17 smithi028 ceph-mon[23744]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-16T08:17:17.433 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:17 smithi028 ceph-mon[23744]: from='mgr.14768 172.21.15.28:0/1901954942' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-16T08:17:17.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:17:17 smithi144 ceph-mon[25668]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-16T08:17:17.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:17:17 smithi144 ceph-mon[25668]: from='mgr.14768 172.21.15.28:0/1901954942' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-16T08:17:18.034 INFO:teuthology.orchestra.run.smithi028.stdout: 5 125799 11.23 MB/sec execute 136 sec latency 53.577 ms 2024-09-16T08:17:18.450 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:18 smithi028 ceph-mon[23744]: pgmap v16: 97 pgs: 97 active+clean; 323 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 9.6 KiB/s rd, 4.7 MiB/s wr, 444 op/s 2024-09-16T08:17:18.450 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:18 smithi028 ceph-mon[23744]: from='mgr.14768 172.21.15.28:0/1901954942' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi144.utzszj", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-16T08:17:18.450 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:18 smithi028 ceph-mon[23744]: from='mgr.14768 ' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi144.utzszj", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-16T08:17:18.450 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:18 smithi028 ceph-mon[23744]: from='mgr.14768 172.21.15.28:0/1901954942' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-16T08:17:18.450 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:18 smithi028 ceph-mon[23744]: from='mgr.14768 172.21.15.28:0/1901954942' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T08:17:18.547 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:17:18 smithi144 ceph-mon[25668]: pgmap v16: 97 pgs: 97 active+clean; 323 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 9.6 KiB/s rd, 4.7 MiB/s wr, 444 op/s 2024-09-16T08:17:18.547 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:17:18 smithi144 ceph-mon[25668]: from='mgr.14768 172.21.15.28:0/1901954942' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi144.utzszj", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-16T08:17:18.548 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:17:18 smithi144 ceph-mon[25668]: from='mgr.14768 ' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi144.utzszj", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-16T08:17:18.548 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:17:18 smithi144 ceph-mon[25668]: from='mgr.14768 172.21.15.28:0/1901954942' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-16T08:17:18.548 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:17:18 smithi144 ceph-mon[25668]: from='mgr.14768 172.21.15.28:0/1901954942' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T08:17:19.034 INFO:teuthology.orchestra.run.smithi028.stdout: 5 126106 11.20 MB/sec execute 137 sec latency 73.624 ms 2024-09-16T08:17:19.458 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:19 smithi028 ceph-mon[23744]: Upgrade: Updating mgr.smithi144.utzszj 2024-09-16T08:17:19.459 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:19 smithi028 ceph-mon[23744]: Deploying daemon mgr.smithi144.utzszj on smithi144 2024-09-16T08:17:19.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:17:19 smithi144 ceph-mon[25668]: Upgrade: Updating mgr.smithi144.utzszj 2024-09-16T08:17:19.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:17:19 smithi144 ceph-mon[25668]: Deploying daemon mgr.smithi144.utzszj on smithi144 2024-09-16T08:17:20.034 INFO:teuthology.orchestra.run.smithi028.stdout: 5 126486 11.22 MB/sec execute 138 sec latency 49.124 ms 2024-09-16T08:17:20.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:17:20 smithi144 ceph-mon[25668]: pgmap v17: 97 pgs: 97 active+clean; 331 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 5.1 MiB/s wr, 502 op/s 2024-09-16T08:17:20.493 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:20 smithi028 ceph-mon[23744]: pgmap v17: 97 pgs: 97 active+clean; 331 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 5.1 MiB/s wr, 502 op/s 2024-09-16T08:17:21.034 INFO:teuthology.orchestra.run.smithi028.stdout: 5 126871 11.19 MB/sec execute 139 sec latency 48.236 ms 2024-09-16T08:17:21.800 INFO:teuthology.orchestra.run.smithi028.stdout:true 2024-09-16T08:17:22.034 INFO:teuthology.orchestra.run.smithi028.stdout: 5 127156 11.19 MB/sec execute 140 sec latency 66.552 ms 2024-09-16T08:17:22.514 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:22 smithi028 ceph-mon[23744]: pgmap v18: 97 pgs: 97 active+clean; 331 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.2 MiB/s wr, 398 op/s 2024-09-16T08:17:22.514 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:22 smithi028 ceph-mon[23744]: from='mgr.14768 172.21.15.28:0/1901954942' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-16T08:17:22.514 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:22 smithi028 ceph-mon[23744]: from='mgr.14768 ' entity='mgr.smithi028.azvyzy' 2024-09-16T08:17:22.514 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:22 smithi028 ceph-mon[23744]: from='mgr.14768 ' entity='mgr.smithi028.azvyzy' 2024-09-16T08:17:22.514 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:22 smithi028 ceph-mon[23744]: from='mgr.14768 172.21.15.28:0/1901954942' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-16T08:17:22.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:17:22 smithi144 ceph-mon[25668]: pgmap v18: 97 pgs: 97 active+clean; 331 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.2 MiB/s wr, 398 op/s 2024-09-16T08:17:22.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:17:22 smithi144 ceph-mon[25668]: from='mgr.14768 172.21.15.28:0/1901954942' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-16T08:17:22.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:17:22 smithi144 ceph-mon[25668]: from='mgr.14768 ' entity='mgr.smithi028.azvyzy' 2024-09-16T08:17:22.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:17:22 smithi144 ceph-mon[25668]: from='mgr.14768 ' entity='mgr.smithi028.azvyzy' 2024-09-16T08:17:22.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:17:22 smithi144 ceph-mon[25668]: from='mgr.14768 172.21.15.28:0/1901954942' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-16T08:17:22.813 INFO:teuthology.orchestra.run.smithi028.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-16T08:17:22.813 INFO:teuthology.orchestra.run.smithi028.stdout:alertmanager.smithi028 smithi028 *:9093,9094 running (103s) 21s ago 10m 19.7M - 0.27.0 11f11916f8cd a14fcb4cb2b6 2024-09-16T08:17:22.813 INFO:teuthology.orchestra.run.smithi028.stdout:crash.smithi028 smithi028 running (10m) 21s ago 10m 6405k - 17.2.0 e1d6a67b021e 69e795c000fd 2024-09-16T08:17:22.813 INFO:teuthology.orchestra.run.smithi028.stdout:crash.smithi144 smithi144 running (10m) 26s ago 10m 6400k - 17.2.0 e1d6a67b021e bbe54b51649c 2024-09-16T08:17:22.813 INFO:teuthology.orchestra.run.smithi028.stdout:grafana.smithi028 smithi028 *:3000 running (94s) 21s ago 10m 45.6M - dad864ee21e9 b55ba8b944a1 2024-09-16T08:17:22.814 INFO:teuthology.orchestra.run.smithi028.stdout:mds.foofs.smithi028.srtgev smithi028 running (4m) 21s ago 4m 36.9M - 17.2.0 e1d6a67b021e 250e7a7f754c 2024-09-16T08:17:22.814 INFO:teuthology.orchestra.run.smithi028.stdout:mds.foofs.smithi144.oozgsi smithi144 running (4m) 26s ago 4m 16.0M - 17.2.0 e1d6a67b021e ecf4c2bb0fdc 2024-09-16T08:17:22.814 INFO:teuthology.orchestra.run.smithi028.stdout:mgr.smithi028.azvyzy smithi028 *:8443,9283,8765 running (49s) 21s ago 11m 494M - 19.3.0-4917-g8293d73f 425e316ba8eb e01d4382d03f 2024-09-16T08:17:22.814 INFO:teuthology.orchestra.run.smithi028.stdout:mgr.smithi144.utzszj smithi144 *:8443,9283,8765 starting - - - - 2024-09-16T08:17:22.814 INFO:teuthology.orchestra.run.smithi028.stdout:mon.smithi028 smithi028 running (11m) 21s ago 12m 58.2M 2048M 17.2.0 e1d6a67b021e a2f688697052 2024-09-16T08:17:22.814 INFO:teuthology.orchestra.run.smithi028.stdout:mon.smithi144 smithi144 running (10m) 26s ago 10m 48.3M 2048M 17.2.0 e1d6a67b021e 1c80e3ff3507 2024-09-16T08:17:22.814 INFO:teuthology.orchestra.run.smithi028.stdout:nfs.foo.0.0.smithi028.dkttgj smithi028 *:2049 running (4m) 21s ago 4m 279M - 4.0 e1d6a67b021e b5dccc45a6ed 2024-09-16T08:17:22.814 INFO:teuthology.orchestra.run.smithi028.stdout:nfs.foo.1.0.smithi144.epipkg smithi144 *:2049 running (4m) 26s ago 4m 84.7M - 4.0 e1d6a67b021e faa89e61f50e 2024-09-16T08:17:22.814 INFO:teuthology.orchestra.run.smithi028.stdout:node-exporter.smithi028 smithi028 *:9100 running (86s) 21s ago 10m 8040k - 1.7.0 72c9c2088986 9e2b8e8a63a9 2024-09-16T08:17:22.814 INFO:teuthology.orchestra.run.smithi028.stdout:node-exporter.smithi144 smithi144 *:9100 running (59s) 26s ago 9m 9541k - 1.7.0 72c9c2088986 78834c5e992f 2024-09-16T08:17:22.814 INFO:teuthology.orchestra.run.smithi028.stdout:osd.0 smithi028 running (9m) 21s ago 9m 446M 1937M 17.2.0 e1d6a67b021e 8184fbc3871f 2024-09-16T08:17:22.814 INFO:teuthology.orchestra.run.smithi028.stdout:osd.1 smithi028 running (8m) 21s ago 8m 364M 1937M 17.2.0 e1d6a67b021e 6911d5c68d54 2024-09-16T08:17:22.814 INFO:teuthology.orchestra.run.smithi028.stdout:osd.2 smithi028 running (8m) 21s ago 8m 282M 1937M 17.2.0 e1d6a67b021e 90b27d8375a2 2024-09-16T08:17:22.814 INFO:teuthology.orchestra.run.smithi028.stdout:osd.3 smithi028 running (7m) 21s ago 7m 414M 1937M 17.2.0 e1d6a67b021e dac477d8b04f 2024-09-16T08:17:22.815 INFO:teuthology.orchestra.run.smithi028.stdout:osd.4 smithi144 running (7m) 26s ago 7m 391M 2704M 17.2.0 e1d6a67b021e 86719c60975c 2024-09-16T08:17:22.815 INFO:teuthology.orchestra.run.smithi028.stdout:osd.5 smithi144 running (7m) 26s ago 7m 439M 2704M 17.2.0 e1d6a67b021e 70452c145c60 2024-09-16T08:17:22.815 INFO:teuthology.orchestra.run.smithi028.stdout:osd.6 smithi144 running (6m) 26s ago 6m 382M 2704M 17.2.0 e1d6a67b021e 7f9981be792f 2024-09-16T08:17:22.815 INFO:teuthology.orchestra.run.smithi028.stdout:osd.7 smithi144 running (6m) 26s ago 6m 522M 2704M 17.2.0 e1d6a67b021e dada74ddbe4c 2024-09-16T08:17:22.815 INFO:teuthology.orchestra.run.smithi028.stdout:prometheus.smithi028 smithi028 *:9095 starting - - - - 2024-09-16T08:17:23.034 INFO:teuthology.orchestra.run.smithi028.stdout: 5 127425 11.19 MB/sec execute 141 sec latency 60.010 ms 2024-09-16T08:17:23.459 INFO:teuthology.orchestra.run.smithi028.stdout:{ 2024-09-16T08:17:23.459 INFO:teuthology.orchestra.run.smithi028.stdout: "mon": { 2024-09-16T08:17:23.459 INFO:teuthology.orchestra.run.smithi028.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-16T08:17:23.460 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-09-16T08:17:23.460 INFO:teuthology.orchestra.run.smithi028.stdout: "mgr": { 2024-09-16T08:17:23.460 INFO:teuthology.orchestra.run.smithi028.stdout: "ceph version 19.3.0-4917-g8293d73f (8293d73f8690540e843a81caec373f9cc29cf705) squid (dev)": 2 2024-09-16T08:17:23.460 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-09-16T08:17:23.460 INFO:teuthology.orchestra.run.smithi028.stdout: "osd": { 2024-09-16T08:17:23.460 INFO:teuthology.orchestra.run.smithi028.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-09-16T08:17:23.460 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-09-16T08:17:23.460 INFO:teuthology.orchestra.run.smithi028.stdout: "mds": { 2024-09-16T08:17:23.460 INFO:teuthology.orchestra.run.smithi028.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-16T08:17:23.460 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-09-16T08:17:23.460 INFO:teuthology.orchestra.run.smithi028.stdout: "overall": { 2024-09-16T08:17:23.460 INFO:teuthology.orchestra.run.smithi028.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 12, 2024-09-16T08:17:23.460 INFO:teuthology.orchestra.run.smithi028.stdout: "ceph version 19.3.0-4917-g8293d73f (8293d73f8690540e843a81caec373f9cc29cf705) squid (dev)": 2 2024-09-16T08:17:23.461 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-09-16T08:17:23.461 INFO:teuthology.orchestra.run.smithi028.stdout:} 2024-09-16T08:17:23.522 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:23 smithi028 ceph-mon[23744]: from='client.14818 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T08:17:23.522 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:23 smithi028 ceph-mon[23744]: from='client.14822 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T08:17:23.522 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:23 smithi028 ceph-mon[23744]: from='mgr.14768 ' entity='mgr.smithi028.azvyzy' 2024-09-16T08:17:23.523 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:23 smithi028 ceph-mon[23744]: from='mgr.14768 ' entity='mgr.smithi028.azvyzy' 2024-09-16T08:17:23.765 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:17:23 smithi144 ceph-mon[25668]: from='client.14818 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T08:17:23.765 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:17:23 smithi144 ceph-mon[25668]: from='client.14822 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T08:17:23.766 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:17:23 smithi144 ceph-mon[25668]: from='mgr.14768 ' entity='mgr.smithi028.azvyzy' 2024-09-16T08:17:23.766 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:17:23 smithi144 ceph-mon[25668]: from='mgr.14768 ' entity='mgr.smithi028.azvyzy' 2024-09-16T08:17:24.010 INFO:teuthology.orchestra.run.smithi028.stdout:{ 2024-09-16T08:17:24.010 INFO:teuthology.orchestra.run.smithi028.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:f7ad55f9ccd17ba66aadb19b4d291e67b60ea24e5d8a0c0e92941fd59c81e4ce", 2024-09-16T08:17:24.010 INFO:teuthology.orchestra.run.smithi028.stdout: "in_progress": true, 2024-09-16T08:17:24.010 INFO:teuthology.orchestra.run.smithi028.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-16T08:17:24.010 INFO:teuthology.orchestra.run.smithi028.stdout: "services_complete": [], 2024-09-16T08:17:24.010 INFO:teuthology.orchestra.run.smithi028.stdout: "progress": "", 2024-09-16T08:17:24.010 INFO:teuthology.orchestra.run.smithi028.stdout: "message": "Currently upgrading mgr daemons", 2024-09-16T08:17:24.011 INFO:teuthology.orchestra.run.smithi028.stdout: "is_paused": false 2024-09-16T08:17:24.011 INFO:teuthology.orchestra.run.smithi028.stdout:} 2024-09-16T08:17:24.034 INFO:teuthology.orchestra.run.smithi028.stdout: 5 127700 11.18 MB/sec execute 142 sec latency 42.119 ms 2024-09-16T08:17:24.549 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:24 smithi028 ceph-mon[23744]: pgmap v19: 97 pgs: 97 active+clean; 326 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 9.2 KiB/s rd, 6.2 MiB/s wr, 614 op/s 2024-09-16T08:17:24.549 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:24 smithi028 ceph-mon[23744]: from='client.14826 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T08:17:24.549 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:24 smithi028 ceph-mon[23744]: from='client.? 172.21.15.28:0/547152503' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-16T08:17:24.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:17:24 smithi144 ceph-mon[25668]: pgmap v19: 97 pgs: 97 active+clean; 326 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 9.2 KiB/s rd, 6.2 MiB/s wr, 614 op/s 2024-09-16T08:17:24.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:17:24 smithi144 ceph-mon[25668]: from='client.14826 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T08:17:24.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:17:24 smithi144 ceph-mon[25668]: from='client.? 172.21.15.28:0/547152503' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-16T08:17:24.661 INFO:teuthology.orchestra.run.smithi028.stdout:HEALTH_OK 2024-09-16T08:17:25.034 INFO:teuthology.orchestra.run.smithi028.stdout: 5 127957 11.16 MB/sec execute 143 sec latency 57.854 ms 2024-09-16T08:17:25.574 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:25 smithi028 ceph-mon[23744]: from='client.14834 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T08:17:25.574 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:25 smithi028 ceph-mon[23744]: from='client.? 172.21.15.28:0/2732358056' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-16T08:17:25.831 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:17:25 smithi144 ceph-mon[25668]: from='client.14834 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T08:17:25.831 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:17:25 smithi144 ceph-mon[25668]: from='client.? 172.21.15.28:0/2732358056' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-16T08:17:26.034 INFO:teuthology.orchestra.run.smithi028.stdout: 5 128202 11.12 MB/sec execute 144 sec latency 59.645 ms 2024-09-16T08:17:26.593 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:26 smithi028 ceph-mon[23744]: pgmap v20: 97 pgs: 97 active+clean; 326 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.1 MiB/s wr, 404 op/s 2024-09-16T08:17:26.831 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:17:26 smithi144 ceph-mon[25668]: pgmap v20: 97 pgs: 97 active+clean; 326 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.1 MiB/s wr, 404 op/s 2024-09-16T08:17:27.034 INFO:teuthology.orchestra.run.smithi028.stdout: 5 128409 11.07 MB/sec execute 145 sec latency 75.380 ms 2024-09-16T08:17:27.616 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:27 smithi028 ceph-mon[23744]: from='mgr.14768 ' entity='mgr.smithi028.azvyzy' 2024-09-16T08:17:27.616 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:27 smithi028 ceph-mon[23744]: from='mgr.14768 ' entity='mgr.smithi028.azvyzy' 2024-09-16T08:17:27.616 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:27 smithi028 ceph-mon[23744]: from='mgr.14768 ' entity='mgr.smithi028.azvyzy' 2024-09-16T08:17:27.616 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:27 smithi028 ceph-mon[23744]: from='mgr.14768 ' entity='mgr.smithi028.azvyzy' 2024-09-16T08:17:27.831 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:17:27 smithi144 ceph-mon[25668]: from='mgr.14768 ' entity='mgr.smithi028.azvyzy' 2024-09-16T08:17:27.832 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:17:27 smithi144 ceph-mon[25668]: from='mgr.14768 ' entity='mgr.smithi028.azvyzy' 2024-09-16T08:17:27.832 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:17:27 smithi144 ceph-mon[25668]: from='mgr.14768 ' entity='mgr.smithi028.azvyzy' 2024-09-16T08:17:27.832 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:17:27 smithi144 ceph-mon[25668]: from='mgr.14768 ' entity='mgr.smithi028.azvyzy' 2024-09-16T08:17:28.035 INFO:teuthology.orchestra.run.smithi028.stdout: 5 128628 11.03 MB/sec execute 146 sec latency 50.991 ms 2024-09-16T08:17:28.629 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:28 smithi028 ceph-mon[23744]: pgmap v21: 97 pgs: 97 active+clean; 327 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 8.4 KiB/s rd, 4.8 MiB/s wr, 496 op/s 2024-09-16T08:17:28.629 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:28 smithi028 ceph-mon[23744]: Standby manager daemon smithi144.utzszj restarted 2024-09-16T08:17:28.629 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:28 smithi028 ceph-mon[23744]: Standby manager daemon smithi144.utzszj started 2024-09-16T08:17:28.630 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:28 smithi028 ceph-mon[23744]: from='mgr.? 172.21.15.144:0/3741024012' entity='mgr.smithi144.utzszj' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi144.utzszj/crt"}]: dispatch 2024-09-16T08:17:28.630 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:28 smithi028 ceph-mon[23744]: from='mgr.? 172.21.15.144:0/3741024012' entity='mgr.smithi144.utzszj' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-16T08:17:28.630 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:28 smithi028 ceph-mon[23744]: from='mgr.? 172.21.15.144:0/3741024012' entity='mgr.smithi144.utzszj' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi144.utzszj/key"}]: dispatch 2024-09-16T08:17:28.630 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:28 smithi028 ceph-mon[23744]: from='mgr.? 172.21.15.144:0/3741024012' entity='mgr.smithi144.utzszj' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-16T08:17:28.831 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:17:28 smithi144 ceph-mon[25668]: pgmap v21: 97 pgs: 97 active+clean; 327 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 8.4 KiB/s rd, 4.8 MiB/s wr, 496 op/s 2024-09-16T08:17:28.832 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:17:28 smithi144 ceph-mon[25668]: Standby manager daemon smithi144.utzszj restarted 2024-09-16T08:17:28.832 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:17:28 smithi144 ceph-mon[25668]: Standby manager daemon smithi144.utzszj started 2024-09-16T08:17:28.832 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:17:28 smithi144 ceph-mon[25668]: from='mgr.? 172.21.15.144:0/3741024012' entity='mgr.smithi144.utzszj' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi144.utzszj/crt"}]: dispatch 2024-09-16T08:17:28.832 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:17:28 smithi144 ceph-mon[25668]: from='mgr.? 172.21.15.144:0/3741024012' entity='mgr.smithi144.utzszj' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-16T08:17:28.832 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:17:28 smithi144 ceph-mon[25668]: from='mgr.? 172.21.15.144:0/3741024012' entity='mgr.smithi144.utzszj' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi144.utzszj/key"}]: dispatch 2024-09-16T08:17:28.832 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:17:28 smithi144 ceph-mon[25668]: from='mgr.? 172.21.15.144:0/3741024012' entity='mgr.smithi144.utzszj' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-16T08:17:29.034 INFO:teuthology.orchestra.run.smithi028.stdout: 5 128924 10.99 MB/sec execute 147 sec latency 57.983 ms 2024-09-16T08:17:29.638 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:29 smithi028 ceph-mon[23744]: mgrmap e35: smithi028.azvyzy(active, since 38s), standbys: smithi144.utzszj 2024-09-16T08:17:29.785 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:17:29 smithi144 ceph-mon[25668]: mgrmap e35: smithi028.azvyzy(active, since 38s), standbys: smithi144.utzszj 2024-09-16T08:17:30.035 INFO:teuthology.orchestra.run.smithi028.stdout: 5 129287 10.96 MB/sec execute 148 sec latency 64.890 ms 2024-09-16T08:17:30.671 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:30 smithi028 ceph-mon[23744]: pgmap v22: 97 pgs: 97 active+clean; 334 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 5.2 KiB/s rd, 5.5 MiB/s wr, 539 op/s 2024-09-16T08:17:30.671 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:30 smithi028 ceph-mon[23744]: from='mgr.14768 ' entity='mgr.smithi028.azvyzy' 2024-09-16T08:17:30.671 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:30 smithi028 ceph-mon[23744]: from='mgr.14768 ' entity='mgr.smithi028.azvyzy' 2024-09-16T08:17:30.681 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:17:30 smithi144 ceph-mon[25668]: pgmap v22: 97 pgs: 97 active+clean; 334 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 5.2 KiB/s rd, 5.5 MiB/s wr, 539 op/s 2024-09-16T08:17:30.682 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:17:30 smithi144 ceph-mon[25668]: from='mgr.14768 ' entity='mgr.smithi028.azvyzy' 2024-09-16T08:17:30.682 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:17:30 smithi144 ceph-mon[25668]: from='mgr.14768 ' entity='mgr.smithi028.azvyzy' 2024-09-16T08:17:31.035 INFO:teuthology.orchestra.run.smithi028.stdout: 5 129645 10.94 MB/sec execute 149 sec latency 52.981 ms 2024-09-16T08:17:32.035 INFO:teuthology.orchestra.run.smithi028.stdout: 5 129988 10.94 MB/sec execute 150 sec latency 95.530 ms 2024-09-16T08:17:32.193 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:32 smithi028 ceph-mon[23744]: pgmap v23: 97 pgs: 97 active+clean; 334 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 5.2 KiB/s rd, 3.9 MiB/s wr, 369 op/s 2024-09-16T08:17:32.193 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:32 smithi028 ceph-mon[23744]: from='mgr.14768 ' entity='mgr.smithi028.azvyzy' 2024-09-16T08:17:32.193 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:32 smithi028 ceph-mon[23744]: from='mgr.14768 ' entity='mgr.smithi028.azvyzy' 2024-09-16T08:17:32.193 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:32 smithi028 ceph-mon[23744]: from='mgr.14768 172.21.15.28:0/1901954942' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T08:17:32.193 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:32 smithi028 ceph-mon[23744]: from='mgr.14768 172.21.15.28:0/1901954942' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-16T08:17:32.193 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:32 smithi028 ceph-mon[23744]: from='mgr.14768 ' entity='mgr.smithi028.azvyzy' 2024-09-16T08:17:32.194 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:32 smithi028 ceph-mon[23744]: from='mgr.14768 ' entity='mgr.smithi028.azvyzy' 2024-09-16T08:17:32.194 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:32 smithi028 ceph-mon[23744]: from='mgr.14768 172.21.15.28:0/1901954942' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-16T08:17:32.194 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:32 smithi028 ceph-mon[23744]: from='mgr.14768 172.21.15.28:0/1901954942' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "versions"}]: dispatch 2024-09-16T08:17:32.194 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:32 smithi028 ceph-mon[23744]: from='mgr.14768 ' entity='mgr.smithi028.azvyzy' 2024-09-16T08:17:32.194 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:32 smithi028 ceph-mon[23744]: from='mgr.14768 172.21.15.28:0/1901954942' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi028.azvyzy"}]: dispatch 2024-09-16T08:17:32.194 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:32 smithi028 ceph-mon[23744]: from='mgr.14768 ' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi028.azvyzy"}]: dispatch 2024-09-16T08:17:32.194 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:32 smithi028 ceph-mon[23744]: from='mgr.14768 ' entity='mgr.smithi028.azvyzy' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi028.azvyzy"}]': finished 2024-09-16T08:17:32.194 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:32 smithi028 ceph-mon[23744]: from='mgr.14768 172.21.15.28:0/1901954942' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi144.utzszj"}]: dispatch 2024-09-16T08:17:32.194 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:32 smithi028 ceph-mon[23744]: from='mgr.14768 ' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi144.utzszj"}]: dispatch 2024-09-16T08:17:32.194 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:32 smithi028 ceph-mon[23744]: from='mgr.14768 ' entity='mgr.smithi028.azvyzy' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi144.utzszj"}]': finished 2024-09-16T08:17:32.194 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:32 smithi028 ceph-mon[23744]: from='mgr.14768 172.21.15.28:0/1901954942' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-09-16T08:17:32.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:17:32 smithi144 ceph-mon[25668]: pgmap v23: 97 pgs: 97 active+clean; 334 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 5.2 KiB/s rd, 3.9 MiB/s wr, 369 op/s 2024-09-16T08:17:32.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:17:32 smithi144 ceph-mon[25668]: from='mgr.14768 ' entity='mgr.smithi028.azvyzy' 2024-09-16T08:17:32.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:17:32 smithi144 ceph-mon[25668]: from='mgr.14768 ' entity='mgr.smithi028.azvyzy' 2024-09-16T08:17:32.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:17:32 smithi144 ceph-mon[25668]: from='mgr.14768 172.21.15.28:0/1901954942' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T08:17:32.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:17:32 smithi144 ceph-mon[25668]: from='mgr.14768 172.21.15.28:0/1901954942' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-16T08:17:32.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:17:32 smithi144 ceph-mon[25668]: from='mgr.14768 ' entity='mgr.smithi028.azvyzy' 2024-09-16T08:17:32.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:17:32 smithi144 ceph-mon[25668]: from='mgr.14768 ' entity='mgr.smithi028.azvyzy' 2024-09-16T08:17:32.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:17:32 smithi144 ceph-mon[25668]: from='mgr.14768 172.21.15.28:0/1901954942' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-16T08:17:32.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:17:32 smithi144 ceph-mon[25668]: from='mgr.14768 172.21.15.28:0/1901954942' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "versions"}]: dispatch 2024-09-16T08:17:32.583 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:17:32 smithi144 ceph-mon[25668]: from='mgr.14768 ' entity='mgr.smithi028.azvyzy' 2024-09-16T08:17:32.583 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:17:32 smithi144 ceph-mon[25668]: from='mgr.14768 172.21.15.28:0/1901954942' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi028.azvyzy"}]: dispatch 2024-09-16T08:17:32.583 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:17:32 smithi144 ceph-mon[25668]: from='mgr.14768 ' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi028.azvyzy"}]: dispatch 2024-09-16T08:17:32.583 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:17:32 smithi144 ceph-mon[25668]: from='mgr.14768 ' entity='mgr.smithi028.azvyzy' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi028.azvyzy"}]': finished 2024-09-16T08:17:32.583 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:17:32 smithi144 ceph-mon[25668]: from='mgr.14768 172.21.15.28:0/1901954942' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi144.utzszj"}]: dispatch 2024-09-16T08:17:32.583 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:17:32 smithi144 ceph-mon[25668]: from='mgr.14768 ' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi144.utzszj"}]: dispatch 2024-09-16T08:17:32.583 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:17:32 smithi144 ceph-mon[25668]: from='mgr.14768 ' entity='mgr.smithi028.azvyzy' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi144.utzszj"}]': finished 2024-09-16T08:17:32.583 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:17:32 smithi144 ceph-mon[25668]: from='mgr.14768 172.21.15.28:0/1901954942' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-09-16T08:17:33.035 INFO:teuthology.orchestra.run.smithi028.stdout: 5 130455 10.95 MB/sec execute 151 sec latency 75.932 ms 2024-09-16T08:17:33.216 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:33 smithi028 ceph-mon[23744]: Upgrade: Setting container_image for all mgr 2024-09-16T08:17:33.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:17:33 smithi144 ceph-mon[25668]: Upgrade: Setting container_image for all mgr 2024-09-16T08:17:34.035 INFO:teuthology.orchestra.run.smithi028.stdout: 5 130793 10.97 MB/sec execute 152 sec latency 63.630 ms 2024-09-16T08:17:34.225 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:34 smithi028 ceph-mon[23744]: pgmap v24: 97 pgs: 97 active+clean; 324 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 8.7 KiB/s rd, 5.5 MiB/s wr, 566 op/s 2024-09-16T08:17:34.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:17:34 smithi144 ceph-mon[25668]: pgmap v24: 97 pgs: 97 active+clean; 324 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 8.7 KiB/s rd, 5.5 MiB/s wr, 566 op/s 2024-09-16T08:17:35.035 INFO:teuthology.orchestra.run.smithi028.stdout: 5 131074 10.97 MB/sec execute 153 sec latency 56.786 ms 2024-09-16T08:17:35.522 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:35 smithi028 ceph-mon[23744]: Upgrade: Updating mon.smithi028 2024-09-16T08:17:35.522 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:35 smithi028 ceph-mon[23744]: from='mgr.14768 ' entity='mgr.smithi028.azvyzy' 2024-09-16T08:17:35.522 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:35 smithi028 ceph-mon[23744]: from='mgr.14768 172.21.15.28:0/1901954942' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-16T08:17:35.522 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:35 smithi028 ceph-mon[23744]: from='mgr.14768 172.21.15.28:0/1901954942' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-16T08:17:35.522 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:35 smithi028 ceph-mon[23744]: from='mgr.14768 172.21.15.28:0/1901954942' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T08:17:35.522 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:35 smithi028 ceph-mon[23744]: Deploying daemon mon.smithi028 on smithi028 2024-09-16T08:17:35.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:17:35 smithi144 ceph-mon[25668]: Upgrade: Updating mon.smithi028 2024-09-16T08:17:35.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:17:35 smithi144 ceph-mon[25668]: from='mgr.14768 ' entity='mgr.smithi028.azvyzy' 2024-09-16T08:17:35.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:17:35 smithi144 ceph-mon[25668]: from='mgr.14768 172.21.15.28:0/1901954942' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-16T08:17:35.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:17:35 smithi144 ceph-mon[25668]: from='mgr.14768 172.21.15.28:0/1901954942' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-16T08:17:35.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:17:35 smithi144 ceph-mon[25668]: from='mgr.14768 172.21.15.28:0/1901954942' entity='mgr.smithi028.azvyzy' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T08:17:35.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:17:35 smithi144 ceph-mon[25668]: Deploying daemon mon.smithi028 on smithi028 2024-09-16T08:17:36.035 INFO:teuthology.orchestra.run.smithi028.stdout: 5 131358 10.95 MB/sec execute 154 sec latency 84.347 ms 2024-09-16T08:17:36.543 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:36 smithi028 ceph-mon[23744]: pgmap v25: 97 pgs: 97 active+clean; 324 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 3.4 MiB/s wr, 351 op/s 2024-09-16T08:17:36.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:17:36 smithi144 ceph-mon[25668]: pgmap v25: 97 pgs: 97 active+clean; 324 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 3.4 MiB/s wr, 351 op/s 2024-09-16T08:17:36.794 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:36 smithi028 systemd[1]: Stopping Ceph mon.smithi028 for 5622b2fe-7402-11ef-bceb-c7b262605968... 2024-09-16T08:17:37.035 INFO:teuthology.orchestra.run.smithi028.stdout: 5 131636 10.92 MB/sec execute 155 sec latency 80.934 ms 2024-09-16T08:17:37.054 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:37 smithi028 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-mon-smithi028[23740]: 2024-09-16T08:17:37.015+0000 7f230ebd6700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.smithi028 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-09-16T08:17:37.054 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:37 smithi028 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-mon-smithi028[23740]: 2024-09-16T08:17:37.015+0000 7f230ebd6700 -1 mon.smithi028@0(leader) e2 *** Got Signal Terminated *** 2024-09-16T08:17:37.054 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:37 smithi028 podman[63959]: 2024-09-16 08:17:37.036285335 +0000 UTC m=+0.170109866 container stop a2f688697052cb0def56c126ac145a22be239a549f59f8d80465942185c8d025 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-5622b2fe-7402-11ef-bceb-c7b262605968-mon-smithi028, release=754, architecture=x86_64, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vcs-type=git, vendor=Red Hat, Inc., vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, RELEASE=HEAD, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.expose-services=, ceph=True, io.openshift.tags=base centos centos-stream, CEPH_POINT_RELEASE=-17.2.0, io.buildah.version=1.19.8, name=centos-stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_REPO=https://github.com/ceph/ceph-container.git, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.component=centos-stream-container, build-date=2022-05-03T08:36:31.336870, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_CLEAN=True, io.k8s.display-name=CentOS Stream 8, version=8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, distribution-scope=public) 2024-09-16T08:17:37.307 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:37 smithi028 podman[63959]: 2024-09-16 08:17:37.062192715 +0000 UTC m=+0.196017240 container died a2f688697052cb0def56c126ac145a22be239a549f59f8d80465942185c8d025 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-5622b2fe-7402-11ef-bceb-c7b262605968-mon-smithi028, GIT_CLEAN=True, name=centos-stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, version=8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., maintainer=Guillaume Abrioux , io.openshift.expose-services=, distribution-scope=public, GIT_BRANCH=HEAD, ceph=True, vendor=Red Hat, Inc., description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, RELEASE=HEAD, release=754, vcs-type=git, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.component=centos-stream-container, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, architecture=x86_64, io.k8s.display-name=CentOS Stream 8, build-date=2022-05-03T08:36:31.336870, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.openshift.tags=base centos centos-stream, CEPH_POINT_RELEASE=-17.2.0, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.buildah.version=1.19.8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac) 2024-09-16T08:17:38.035 INFO:teuthology.orchestra.run.smithi028.stdout: 5 131857 10.89 MB/sec execute 156 sec latency 96.475 ms 2024-09-16T08:17:38.100 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:37 smithi028 podman[63959]: 2024-09-16 08:17:37.965756627 +0000 UTC m=+1.099581147 container cleanup a2f688697052cb0def56c126ac145a22be239a549f59f8d80465942185c8d025 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-5622b2fe-7402-11ef-bceb-c7b262605968-mon-smithi028, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, ceph=True, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, architecture=x86_64, GIT_BRANCH=HEAD, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.openshift.tags=base centos centos-stream, GIT_REPO=https://github.com/ceph/ceph-container.git, release=754, vendor=Red Hat, Inc., maintainer=Guillaume Abrioux , vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, version=8, vcs-type=git, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., distribution-scope=public, CEPH_POINT_RELEASE=-17.2.0, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_CLEAN=True, io.openshift.expose-services=, RELEASE=HEAD, name=centos-stream, com.redhat.component=centos-stream-container, build-date=2022-05-03T08:36:31.336870, io.k8s.display-name=CentOS Stream 8, io.buildah.version=1.19.8) 2024-09-16T08:17:38.100 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:37 smithi028 bash[63959]: ceph-5622b2fe-7402-11ef-bceb-c7b262605968-mon-smithi028 2024-09-16T08:17:38.626 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:38 smithi028 podman[63972]: 2024-09-16 08:17:38.408278432 +0000 UTC m=+1.365303452 container remove a2f688697052cb0def56c126ac145a22be239a549f59f8d80465942185c8d025 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-5622b2fe-7402-11ef-bceb-c7b262605968-mon-smithi028, io.k8s.display-name=CentOS Stream 8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, name=centos-stream, CEPH_POINT_RELEASE=-17.2.0, maintainer=Guillaume Abrioux , GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_CLEAN=True, io.openshift.expose-services=, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, ceph=True, architecture=x86_64, RELEASE=HEAD, vcs-type=git, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vendor=Red Hat, Inc., GIT_REPO=https://github.com/ceph/ceph-container.git, io.buildah.version=1.19.8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, version=8, distribution-scope=public, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, build-date=2022-05-03T08:36:31.336870, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.openshift.tags=base centos centos-stream, com.redhat.component=centos-stream-container, GIT_BRANCH=HEAD, release=754) 2024-09-16T08:17:38.877 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:38 smithi028 systemd[1]: ceph-5622b2fe-7402-11ef-bceb-c7b262605968@mon.smithi028.service: Deactivated successfully. 2024-09-16T08:17:38.877 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:38 smithi028 systemd[1]: Stopped Ceph mon.smithi028 for 5622b2fe-7402-11ef-bceb-c7b262605968. 2024-09-16T08:17:38.877 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:38 smithi028 systemd[1]: ceph-5622b2fe-7402-11ef-bceb-c7b262605968@mon.smithi028.service: Consumed 17.759s CPU time. 2024-09-16T08:17:39.035 INFO:teuthology.orchestra.run.smithi028.stdout: 5 132087 10.86 MB/sec execute 157 sec latency 60.373 ms 2024-09-16T08:17:39.131 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:39 smithi028 systemd[1]: Starting Ceph mon.smithi028 for 5622b2fe-7402-11ef-bceb-c7b262605968... 2024-09-16T08:17:39.382 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:39 smithi028 podman[64097]: 2024-09-16 08:17:39.272215478 +0000 UTC m=+0.018899169 image pull 425e316ba8ebf74f04aa3640f7deb4fa3f293a78d0c8d097cb679eaf86ddca6c quay.ceph.io/ceph-ci/ceph@sha256:f7ad55f9ccd17ba66aadb19b4d291e67b60ea24e5d8a0c0e92941fd59c81e4ce 2024-09-16T08:17:39.651 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:39 smithi028 podman[64097]: 2024-09-16 08:17:39.409986246 +0000 UTC m=+0.156669917 container create 14efcedc0088717f01c964e67e71441d1547d4705d5236fd6e00c0beff69d513 (image=quay.ceph.io/ceph-ci/ceph@sha256:f7ad55f9ccd17ba66aadb19b4d291e67b60ea24e5d8a0c0e92941fd59c81e4ce, name=ceph-5622b2fe-7402-11ef-bceb-c7b262605968-mon-smithi028, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , RELEASE=wip-guits-main-2024-09-13-1248-8293d73, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD) 2024-09-16T08:17:39.904 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:39 smithi028 podman[64097]: 2024-09-16 08:17:39.781081994 +0000 UTC m=+0.527765680 container init 14efcedc0088717f01c964e67e71441d1547d4705d5236fd6e00c0beff69d513 (image=quay.ceph.io/ceph-ci/ceph@sha256:f7ad55f9ccd17ba66aadb19b4d291e67b60ea24e5d8a0c0e92941fd59c81e4ce, name=ceph-5622b2fe-7402-11ef-bceb-c7b262605968-mon-smithi028, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, RELEASE=wip-guits-main-2024-09-13-1248-8293d73, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS) 2024-09-16T08:17:39.904 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:39 smithi028 podman[64097]: 2024-09-16 08:17:39.790751147 +0000 UTC m=+0.537434826 container start 14efcedc0088717f01c964e67e71441d1547d4705d5236fd6e00c0beff69d513 (image=quay.ceph.io/ceph-ci/ceph@sha256:f7ad55f9ccd17ba66aadb19b4d291e67b60ea24e5d8a0c0e92941fd59c81e4ce, name=ceph-5622b2fe-7402-11ef-bceb-c7b262605968-mon-smithi028, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.license=GPLv2, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=wip-guits-main-2024-09-13-1248-8293d73, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0) 2024-09-16T08:17:39.905 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:39 smithi028 ceph-mon[64115]: set uid:gid to 167:167 (ceph:ceph) 2024-09-16T08:17:39.905 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:39 smithi028 ceph-mon[64115]: ceph version 19.3.0-4917-g8293d73f (8293d73f8690540e843a81caec373f9cc29cf705) squid (dev), process ceph-mon, pid 2 2024-09-16T08:17:39.905 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:39 smithi028 ceph-mon[64115]: pidfile_write: ignore empty --pid-file 2024-09-16T08:17:39.905 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:39 smithi028 ceph-mon[64115]: load: jerasure load: lrc 2024-09-16T08:17:39.905 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:39 smithi028 ceph-mon[64115]: rocksdb: RocksDB version: 7.9.2 2024-09-16T08:17:39.905 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:39 smithi028 ceph-mon[64115]: rocksdb: Git sha 0 2024-09-16T08:17:39.905 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:39 smithi028 ceph-mon[64115]: rocksdb: Compile date 2024-09-13 14:50:41 2024-09-16T08:17:39.905 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:39 smithi028 ceph-mon[64115]: rocksdb: DB SUMMARY 2024-09-16T08:17:39.905 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:39 smithi028 ceph-mon[64115]: rocksdb: DB Session ID: OTUXEGBLNUST655O990G 2024-09-16T08:17:39.905 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:39 smithi028 ceph-mon[64115]: rocksdb: CURRENT file: CURRENT 2024-09-16T08:17:39.906 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:39 smithi028 ceph-mon[64115]: rocksdb: IDENTITY file: IDENTITY 2024-09-16T08:17:39.906 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:39 smithi028 ceph-mon[64115]: rocksdb: MANIFEST file: MANIFEST-000015 size: 978 Bytes 2024-09-16T08:17:39.906 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:39 smithi028 ceph-mon[64115]: rocksdb: SST files in /var/lib/ceph/mon/ceph-smithi028/store.db dir, Total Num: 1, files: 000030.sst 2024-09-16T08:17:39.906 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:39 smithi028 ceph-mon[64115]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-smithi028/store.db: 000028.log size: 2487803 ; 2024-09-16T08:17:39.906 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:39 smithi028 ceph-mon[64115]: rocksdb: Options.error_if_exists: 0 2024-09-16T08:17:39.906 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:39 smithi028 ceph-mon[64115]: rocksdb: Options.create_if_missing: 0 2024-09-16T08:17:39.906 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:39 smithi028 ceph-mon[64115]: rocksdb: Options.paranoid_checks: 1 2024-09-16T08:17:39.906 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:39 smithi028 ceph-mon[64115]: rocksdb: Options.flush_verify_memtable_count: 1 2024-09-16T08:17:39.906 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:39 smithi028 ceph-mon[64115]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-09-16T08:17:39.906 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:39 smithi028 ceph-mon[64115]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-09-16T08:17:39.906 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:39 smithi028 ceph-mon[64115]: rocksdb: Options.env: 0x55d703264120 2024-09-16T08:17:39.906 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:39 smithi028 ceph-mon[64115]: rocksdb: Options.fs: PosixFileSystem 2024-09-16T08:17:39.906 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:39 smithi028 ceph-mon[64115]: rocksdb: Options.info_log: 0x55d7045e78e0 2024-09-16T08:17:39.906 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:39 smithi028 ceph-mon[64115]: rocksdb: Options.max_file_opening_threads: 16 2024-09-16T08:17:39.907 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:39 smithi028 ceph-mon[64115]: rocksdb: Options.statistics: (nil) 2024-09-16T08:17:39.907 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:39 smithi028 ceph-mon[64115]: rocksdb: Options.use_fsync: 0 2024-09-16T08:17:39.907 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:39 smithi028 ceph-mon[64115]: rocksdb: Options.max_log_file_size: 0 2024-09-16T08:17:39.907 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:39 smithi028 ceph-mon[64115]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-09-16T08:17:39.908 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:39 smithi028 ceph-mon[64115]: rocksdb: Options.log_file_time_to_roll: 0 2024-09-16T08:17:39.908 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:39 smithi028 ceph-mon[64115]: rocksdb: Options.keep_log_file_num: 1000 2024-09-16T08:17:39.908 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:39 smithi028 ceph-mon[64115]: rocksdb: Options.recycle_log_file_num: 0 2024-09-16T08:17:39.908 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:39 smithi028 ceph-mon[64115]: rocksdb: Options.allow_fallocate: 1 2024-09-16T08:17:39.908 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:39 smithi028 ceph-mon[64115]: rocksdb: Options.allow_mmap_reads: 0 2024-09-16T08:17:39.908 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:39 smithi028 ceph-mon[64115]: rocksdb: Options.allow_mmap_writes: 0 2024-09-16T08:17:39.908 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:39 smithi028 ceph-mon[64115]: rocksdb: Options.use_direct_reads: 0 2024-09-16T08:17:39.908 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:39 smithi028 ceph-mon[64115]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-09-16T08:17:39.908 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:39 smithi028 ceph-mon[64115]: rocksdb: Options.create_missing_column_families: 0 2024-09-16T08:17:39.908 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:39 smithi028 ceph-mon[64115]: rocksdb: Options.db_log_dir: 2024-09-16T08:17:39.908 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:39 smithi028 ceph-mon[64115]: rocksdb: Options.wal_dir: 2024-09-16T08:17:39.908 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:39 smithi028 ceph-mon[64115]: rocksdb: Options.table_cache_numshardbits: 6 2024-09-16T08:17:39.908 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:39 smithi028 ceph-mon[64115]: rocksdb: Options.WAL_ttl_seconds: 0 2024-09-16T08:17:39.908 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:39 smithi028 ceph-mon[64115]: rocksdb: Options.WAL_size_limit_MB: 0 2024-09-16T08:17:39.909 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:39 smithi028 ceph-mon[64115]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-09-16T08:17:39.909 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:39 smithi028 ceph-mon[64115]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-09-16T08:17:39.909 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:39 smithi028 ceph-mon[64115]: rocksdb: Options.is_fd_close_on_exec: 1 2024-09-16T08:17:39.909 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:39 smithi028 ceph-mon[64115]: rocksdb: Options.advise_random_on_open: 1 2024-09-16T08:17:39.909 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:39 smithi028 ceph-mon[64115]: rocksdb: Options.db_write_buffer_size: 0 2024-09-16T08:17:39.909 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:39 smithi028 ceph-mon[64115]: rocksdb: Options.write_buffer_manager: 0x55d7045ebb80 2024-09-16T08:17:39.909 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:39 smithi028 ceph-mon[64115]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-09-16T08:17:39.909 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:39 smithi028 ceph-mon[64115]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-09-16T08:17:39.910 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:39 smithi028 ceph-mon[64115]: rocksdb: Options.use_adaptive_mutex: 0 2024-09-16T08:17:39.910 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:39 smithi028 ceph-mon[64115]: rocksdb: Options.rate_limiter: (nil) 2024-09-16T08:17:39.910 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:39 smithi028 ceph-mon[64115]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-09-16T08:17:39.910 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:39 smithi028 ceph-mon[64115]: rocksdb: Options.wal_recovery_mode: 2 2024-09-16T08:17:39.910 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:39 smithi028 ceph-mon[64115]: rocksdb: Options.enable_thread_tracking: 0 2024-09-16T08:17:39.910 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:39 smithi028 ceph-mon[64115]: rocksdb: Options.enable_pipelined_write: 0 2024-09-16T08:17:39.910 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:39 smithi028 ceph-mon[64115]: rocksdb: Options.unordered_write: 0 2024-09-16T08:17:39.910 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:39 smithi028 ceph-mon[64115]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-09-16T08:17:39.911 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:39 smithi028 ceph-mon[64115]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-09-16T08:17:39.911 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:39 smithi028 ceph-mon[64115]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-09-16T08:17:39.911 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:39 smithi028 ceph-mon[64115]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-09-16T08:17:39.911 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:39 smithi028 ceph-mon[64115]: rocksdb: Options.row_cache: None 2024-09-16T08:17:39.911 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:39 smithi028 ceph-mon[64115]: rocksdb: Options.wal_filter: None 2024-09-16T08:17:39.911 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:39 smithi028 ceph-mon[64115]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-09-16T08:17:39.911 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:39 smithi028 ceph-mon[64115]: rocksdb: Options.allow_ingest_behind: 0 2024-09-16T08:17:39.911 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:39 smithi028 ceph-mon[64115]: rocksdb: Options.two_write_queues: 0 2024-09-16T08:17:39.911 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:39 smithi028 ceph-mon[64115]: rocksdb: Options.manual_wal_flush: 0 2024-09-16T08:17:39.911 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:39 smithi028 ceph-mon[64115]: rocksdb: Options.wal_compression: 0 2024-09-16T08:17:39.911 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:39 smithi028 ceph-mon[64115]: rocksdb: Options.atomic_flush: 0 2024-09-16T08:17:39.911 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:39 smithi028 ceph-mon[64115]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-09-16T08:17:39.911 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:39 smithi028 ceph-mon[64115]: rocksdb: Options.persist_stats_to_disk: 0 2024-09-16T08:17:39.912 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:39 smithi028 ceph-mon[64115]: rocksdb: Options.write_dbid_to_manifest: 0 2024-09-16T08:17:39.912 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:39 smithi028 ceph-mon[64115]: rocksdb: Options.log_readahead_size: 0 2024-09-16T08:17:39.912 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:39 smithi028 ceph-mon[64115]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-09-16T08:17:39.912 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:39 smithi028 ceph-mon[64115]: rocksdb: Options.best_efforts_recovery: 0 2024-09-16T08:17:39.912 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:39 smithi028 ceph-mon[64115]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-09-16T08:17:39.912 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:39 smithi028 ceph-mon[64115]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-09-16T08:17:39.912 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:39 smithi028 ceph-mon[64115]: rocksdb: Options.allow_data_in_errors: 0 2024-09-16T08:17:39.912 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:39 smithi028 ceph-mon[64115]: rocksdb: Options.db_host_id: __hostname__ 2024-09-16T08:17:39.912 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:39 smithi028 ceph-mon[64115]: rocksdb: Options.enforce_single_del_contracts: true 2024-09-16T08:17:39.912 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:39 smithi028 ceph-mon[64115]: rocksdb: Options.max_background_jobs: 2 2024-09-16T08:17:39.912 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:39 smithi028 ceph-mon[64115]: rocksdb: Options.max_background_compactions: -1 2024-09-16T08:17:39.912 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:39 smithi028 ceph-mon[64115]: rocksdb: Options.max_subcompactions: 1 2024-09-16T08:17:39.913 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:39 smithi028 ceph-mon[64115]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-09-16T08:17:39.913 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:39 smithi028 ceph-mon[64115]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-09-16T08:17:39.913 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:39 smithi028 ceph-mon[64115]: rocksdb: Options.delayed_write_rate : 16777216 2024-09-16T08:17:39.913 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:39 smithi028 ceph-mon[64115]: rocksdb: Options.max_total_wal_size: 0 2024-09-16T08:17:39.913 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:39 smithi028 ceph-mon[64115]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-09-16T08:17:39.913 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:39 smithi028 ceph-mon[64115]: rocksdb: Options.stats_dump_period_sec: 600 2024-09-16T08:17:39.913 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:39 smithi028 ceph-mon[64115]: rocksdb: Options.stats_persist_period_sec: 600 2024-09-16T08:17:39.913 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:39 smithi028 ceph-mon[64115]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-09-16T08:17:39.913 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:39 smithi028 ceph-mon[64115]: rocksdb: Options.max_open_files: -1 2024-09-16T08:17:39.913 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:39 smithi028 ceph-mon[64115]: rocksdb: Options.bytes_per_sync: 0 2024-09-16T08:17:39.913 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:39 smithi028 ceph-mon[64115]: rocksdb: Options.wal_bytes_per_sync: 0 2024-09-16T08:17:39.913 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:39 smithi028 ceph-mon[64115]: rocksdb: Options.strict_bytes_per_sync: 0 2024-09-16T08:17:39.913 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:39 smithi028 ceph-mon[64115]: rocksdb: Options.compaction_readahead_size: 0 2024-09-16T08:17:39.913 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:39 smithi028 ceph-mon[64115]: rocksdb: Options.max_background_flushes: -1 2024-09-16T08:17:39.914 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:39 smithi028 ceph-mon[64115]: rocksdb: Compression algorithms supported: 2024-09-16T08:17:39.914 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:39 smithi028 ceph-mon[64115]: rocksdb: kZSTD supported: 0 2024-09-16T08:17:39.914 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:39 smithi028 ceph-mon[64115]: rocksdb: kXpressCompression supported: 0 2024-09-16T08:17:39.914 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:39 smithi028 ceph-mon[64115]: rocksdb: kBZip2Compression supported: 0 2024-09-16T08:17:39.914 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:39 smithi028 ceph-mon[64115]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-09-16T08:17:39.914 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:39 smithi028 ceph-mon[64115]: rocksdb: kLZ4Compression supported: 1 2024-09-16T08:17:39.914 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:39 smithi028 ceph-mon[64115]: rocksdb: kZlibCompression supported: 1 2024-09-16T08:17:39.915 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:39 smithi028 ceph-mon[64115]: rocksdb: kLZ4HCCompression supported: 1 2024-09-16T08:17:39.915 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:39 smithi028 ceph-mon[64115]: rocksdb: kSnappyCompression supported: 1 2024-09-16T08:17:39.915 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:39 smithi028 ceph-mon[64115]: rocksdb: Fast CRC32 supported: Supported on x86 2024-09-16T08:17:39.916 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:39 smithi028 ceph-mon[64115]: rocksdb: DMutex implementation: pthread_mutex_t 2024-09-16T08:17:39.916 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:39 smithi028 ceph-mon[64115]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-smithi028/store.db/MANIFEST-000015 2024-09-16T08:17:39.916 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:39 smithi028 ceph-mon[64115]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-09-16T08:17:39.916 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:39 smithi028 ceph-mon[64115]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-09-16T08:17:39.916 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:39 smithi028 ceph-mon[64115]: rocksdb: Options.merge_operator: 2024-09-16T08:17:39.916 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:39 smithi028 ceph-mon[64115]: rocksdb: Options.compaction_filter: None 2024-09-16T08:17:39.917 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:39 smithi028 ceph-mon[64115]: rocksdb: Options.compaction_filter_factory: None 2024-09-16T08:17:39.917 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:39 smithi028 ceph-mon[64115]: rocksdb: Options.sst_partitioner_factory: None 2024-09-16T08:17:39.917 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:39 smithi028 ceph-mon[64115]: rocksdb: Options.memtable_factory: SkipListFactory 2024-09-16T08:17:39.917 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:39 smithi028 ceph-mon[64115]: rocksdb: Options.table_factory: BlockBasedTable 2024-09-16T08:17:39.917 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:39 smithi028 ceph-mon[64115]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55d7045e7540) 2024-09-16T08:17:39.917 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout: cache_index_and_filter_blocks: 1 2024-09-16T08:17:39.917 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-09-16T08:17:39.917 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-09-16T08:17:39.917 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout: pin_top_level_index_and_filter: 1 2024-09-16T08:17:39.917 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout: index_type: 0 2024-09-16T08:17:39.917 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout: data_block_index_type: 0 2024-09-16T08:17:39.917 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout: index_shortening: 1 2024-09-16T08:17:39.917 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout: data_block_hash_table_util_ratio: 0.750000 2024-09-16T08:17:39.918 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout: checksum: 4 2024-09-16T08:17:39.918 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout: no_block_cache: 0 2024-09-16T08:17:39.918 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout: block_cache: 0x55d7045db610 2024-09-16T08:17:39.918 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout: block_cache_name: BinnedLRUCache 2024-09-16T08:17:39.918 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout: block_cache_options: 2024-09-16T08:17:39.918 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout: capacity : 536870912 2024-09-16T08:17:39.918 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout: num_shard_bits : 4 2024-09-16T08:17:39.918 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout: strict_capacity_limit : 0 2024-09-16T08:17:39.918 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout: high_pri_pool_ratio: 0.000 2024-09-16T08:17:39.918 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout: block_cache_compressed: (nil) 2024-09-16T08:17:39.918 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout: persistent_cache: (nil) 2024-09-16T08:17:39.918 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout: block_size: 4096 2024-09-16T08:17:39.918 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout: block_size_deviation: 10 2024-09-16T08:17:39.918 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout: block_restart_interval: 16 2024-09-16T08:17:39.919 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout: index_block_restart_interval: 1 2024-09-16T08:17:39.919 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout: metadata_block_size: 4096 2024-09-16T08:17:39.919 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout: partition_filters: 0 2024-09-16T08:17:39.919 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout: use_delta_encoding: 1 2024-09-16T08:17:39.919 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout: filter_policy: bloomfilter 2024-09-16T08:17:39.919 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout: whole_key_filtering: 1 2024-09-16T08:17:39.919 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout: verify_compression: 0 2024-09-16T08:17:39.919 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout: read_amp_bytes_per_bit: 0 2024-09-16T08:17:39.919 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout: format_version: 5 2024-09-16T08:17:39.919 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout: enable_index_compression: 1 2024-09-16T08:17:39.919 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout: block_align: 0 2024-09-16T08:17:39.920 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout: max_auto_readahead_size: 262144 2024-09-16T08:17:39.920 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout: prepopulate_block_cache: 0 2024-09-16T08:17:39.920 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout: initial_auto_readahead_size: 8192 2024-09-16T08:17:39.920 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout: num_file_reads_for_auto_readahead: 2 2024-09-16T08:17:39.920 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:39 smithi028 ceph-mon[64115]: rocksdb: Options.write_buffer_size: 33554432 2024-09-16T08:17:39.920 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:39 smithi028 ceph-mon[64115]: rocksdb: Options.max_write_buffer_number: 2 2024-09-16T08:17:39.920 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:39 smithi028 ceph-mon[64115]: rocksdb: Options.compression: NoCompression 2024-09-16T08:17:39.920 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:39 smithi028 ceph-mon[64115]: rocksdb: Options.bottommost_compression: Disabled 2024-09-16T08:17:39.920 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:39 smithi028 ceph-mon[64115]: rocksdb: Options.prefix_extractor: nullptr 2024-09-16T08:17:39.920 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:39 smithi028 ceph-mon[64115]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-09-16T08:17:39.920 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:39 smithi028 ceph-mon[64115]: rocksdb: Options.num_levels: 7 2024-09-16T08:17:39.920 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:39 smithi028 ceph-mon[64115]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-09-16T08:17:39.920 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:39 smithi028 ceph-mon[64115]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-09-16T08:17:39.921 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:39 smithi028 ceph-mon[64115]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-09-16T08:17:39.921 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:39 smithi028 ceph-mon[64115]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-09-16T08:17:39.921 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:39 smithi028 ceph-mon[64115]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-09-16T08:17:39.921 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:39 smithi028 ceph-mon[64115]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-09-16T08:17:39.921 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:39 smithi028 ceph-mon[64115]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-09-16T08:17:39.921 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:39 smithi028 ceph-mon[64115]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-09-16T08:17:39.921 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:39 smithi028 ceph-mon[64115]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-09-16T08:17:39.921 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:39 smithi028 ceph-mon[64115]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-09-16T08:17:39.921 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:39 smithi028 ceph-mon[64115]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-09-16T08:17:39.921 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:39 smithi028 ceph-mon[64115]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-09-16T08:17:39.921 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:39 smithi028 ceph-mon[64115]: rocksdb: Options.compression_opts.window_bits: -14 2024-09-16T08:17:39.921 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:39 smithi028 ceph-mon[64115]: rocksdb: Options.compression_opts.level: 32767 2024-09-16T08:17:39.921 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:39 smithi028 ceph-mon[64115]: rocksdb: Options.compression_opts.strategy: 0 2024-09-16T08:17:39.921 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:39 smithi028 ceph-mon[64115]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-09-16T08:17:39.921 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:39 smithi028 ceph-mon[64115]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-09-16T08:17:39.922 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:39 smithi028 ceph-mon[64115]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-09-16T08:17:39.922 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:39 smithi028 ceph-mon[64115]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-09-16T08:17:39.922 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:39 smithi028 ceph-mon[64115]: rocksdb: Options.compression_opts.enabled: false 2024-09-16T08:17:39.922 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:39 smithi028 ceph-mon[64115]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-09-16T08:17:39.922 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:39 smithi028 ceph-mon[64115]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-09-16T08:17:39.922 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:39 smithi028 ceph-mon[64115]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-09-16T08:17:39.922 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:39 smithi028 ceph-mon[64115]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-09-16T08:17:39.922 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:39 smithi028 ceph-mon[64115]: rocksdb: Options.target_file_size_base: 67108864 2024-09-16T08:17:39.922 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:39 smithi028 ceph-mon[64115]: rocksdb: Options.target_file_size_multiplier: 1 2024-09-16T08:17:39.922 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:39 smithi028 ceph-mon[64115]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-09-16T08:17:39.922 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:39 smithi028 ceph-mon[64115]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-09-16T08:17:39.922 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:39 smithi028 ceph-mon[64115]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-09-16T08:17:39.922 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:39 smithi028 ceph-mon[64115]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-09-16T08:17:39.923 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:39 smithi028 ceph-mon[64115]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-09-16T08:17:39.923 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:39 smithi028 ceph-mon[64115]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-09-16T08:17:39.923 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:39 smithi028 ceph-mon[64115]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-09-16T08:17:39.923 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:39 smithi028 ceph-mon[64115]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-09-16T08:17:39.923 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:39 smithi028 ceph-mon[64115]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-09-16T08:17:39.923 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:39 smithi028 ceph-mon[64115]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-09-16T08:17:39.923 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:39 smithi028 ceph-mon[64115]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-09-16T08:17:39.923 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:39 smithi028 ceph-mon[64115]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-09-16T08:17:39.924 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:39 smithi028 ceph-mon[64115]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-09-16T08:17:39.924 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:39 smithi028 ceph-mon[64115]: rocksdb: Options.arena_block_size: 1048576 2024-09-16T08:17:39.924 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:39 smithi028 ceph-mon[64115]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-09-16T08:17:39.924 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:39 smithi028 ceph-mon[64115]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-09-16T08:17:39.924 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:39 smithi028 ceph-mon[64115]: rocksdb: Options.disable_auto_compactions: 0 2024-09-16T08:17:39.924 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:39 smithi028 ceph-mon[64115]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-09-16T08:17:39.924 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:39 smithi028 ceph-mon[64115]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-09-16T08:17:39.924 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:39 smithi028 ceph-mon[64115]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-09-16T08:17:39.924 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:39 smithi028 ceph-mon[64115]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-09-16T08:17:39.924 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:39 smithi028 ceph-mon[64115]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-09-16T08:17:39.924 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:39 smithi028 ceph-mon[64115]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-09-16T08:17:39.924 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:39 smithi028 ceph-mon[64115]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-09-16T08:17:39.924 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:39 smithi028 ceph-mon[64115]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-09-16T08:17:39.924 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:39 smithi028 ceph-mon[64115]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-09-16T08:17:39.925 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:39 smithi028 ceph-mon[64115]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-09-16T08:17:39.925 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:39 smithi028 ceph-mon[64115]: rocksdb: Options.table_properties_collectors: CompactOnDeletionCollector (Sliding window size = 32768 Deletion trigger = 16384 Deletion ratio = 0); 2024-09-16T08:17:39.925 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:39 smithi028 ceph-mon[64115]: rocksdb: Options.inplace_update_support: 0 2024-09-16T08:17:39.925 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:39 smithi028 ceph-mon[64115]: rocksdb: Options.inplace_update_num_locks: 10000 2024-09-16T08:17:39.925 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:39 smithi028 ceph-mon[64115]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-09-16T08:17:39.925 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:39 smithi028 ceph-mon[64115]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-09-16T08:17:39.925 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:39 smithi028 ceph-mon[64115]: rocksdb: Options.memtable_huge_page_size: 0 2024-09-16T08:17:39.925 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:39 smithi028 ceph-mon[64115]: rocksdb: Options.bloom_locality: 0 2024-09-16T08:17:39.925 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:39 smithi028 ceph-mon[64115]: rocksdb: Options.max_successive_merges: 0 2024-09-16T08:17:39.925 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:39 smithi028 ceph-mon[64115]: rocksdb: Options.optimize_filters_for_hits: 0 2024-09-16T08:17:39.925 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:39 smithi028 ceph-mon[64115]: rocksdb: Options.paranoid_file_checks: 0 2024-09-16T08:17:39.925 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:39 smithi028 ceph-mon[64115]: rocksdb: Options.force_consistency_checks: 1 2024-09-16T08:17:39.925 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:39 smithi028 ceph-mon[64115]: rocksdb: Options.report_bg_io_stats: 0 2024-09-16T08:17:39.925 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:39 smithi028 ceph-mon[64115]: rocksdb: Options.ttl: 2592000 2024-09-16T08:17:39.926 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:39 smithi028 ceph-mon[64115]: rocksdb: Options.periodic_compaction_seconds: 0 2024-09-16T08:17:39.926 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:39 smithi028 ceph-mon[64115]: rocksdb: Options.preclude_last_level_data_seconds: 0 2024-09-16T08:17:39.926 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:39 smithi028 ceph-mon[64115]: rocksdb: Options.preserve_internal_time_seconds: 0 2024-09-16T08:17:39.926 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:39 smithi028 ceph-mon[64115]: rocksdb: Options.enable_blob_files: false 2024-09-16T08:17:39.926 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:39 smithi028 ceph-mon[64115]: rocksdb: Options.min_blob_size: 0 2024-09-16T08:17:39.926 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:39 smithi028 ceph-mon[64115]: rocksdb: Options.blob_file_size: 268435456 2024-09-16T08:17:39.926 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:39 smithi028 ceph-mon[64115]: rocksdb: Options.blob_compression_type: NoCompression 2024-09-16T08:17:39.926 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:39 smithi028 ceph-mon[64115]: rocksdb: Options.enable_blob_garbage_collection: false 2024-09-16T08:17:39.926 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:39 smithi028 ceph-mon[64115]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-09-16T08:17:39.926 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:39 smithi028 ceph-mon[64115]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-09-16T08:17:39.927 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:39 smithi028 ceph-mon[64115]: rocksdb: Options.blob_compaction_readahead_size: 0 2024-09-16T08:17:39.927 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:39 smithi028 ceph-mon[64115]: rocksdb: Options.blob_file_starting_level: 0 2024-09-16T08:17:39.927 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:39 smithi028 ceph-mon[64115]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-09-16T08:17:39.927 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:39 smithi028 ceph-mon[64115]: rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 30.sst 2024-09-16T08:17:39.927 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:39 smithi028 ceph-mon[64115]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-09-16T08:17:39.927 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:39 smithi028 ceph-mon[64115]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-smithi028/store.db/MANIFEST-000015 succeeded,manifest_file_number is 15, next_file_number is 32, last_sequence is 11855, log_number is 28,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-09-16T08:17:39.928 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:39 smithi028 ceph-mon[64115]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 28 2024-09-16T08:17:39.928 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:39 smithi028 ceph-mon[64115]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: ec038f90-c9c7-40e5-a834-3ab51b72ee66 2024-09-16T08:17:39.928 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:39 smithi028 ceph-mon[64115]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726474659826898, "job": 1, "event": "recovery_started", "wal_files": [28]} 2024-09-16T08:17:39.928 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:39 smithi028 ceph-mon[64115]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #28 mode 2 2024-09-16T08:17:39.928 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:39 smithi028 ceph-mon[64115]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726474659835158, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 33, "file_size": 2306412, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 11856, "largest_seqno": 12783, "table_properties": {"data_size": 2301563, "index_size": 2781, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 1029, "raw_key_size": 9400, "raw_average_key_size": 23, "raw_value_size": 2292733, "raw_average_value_size": 5675, "num_data_blocks": 127, "num_entries": 404, "num_filter_entries": 404, "num_deletions": 2, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[CompactOnDeletionCollector]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1726474659, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "ec038f90-c9c7-40e5-a834-3ab51b72ee66", "db_session_id": "OTUXEGBLNUST655O990G", "orig_file_number": 33, "seqno_to_time_mapping": "N/A"}} 2024-09-16T08:17:39.928 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:39 smithi028 ceph-mon[64115]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726474659835319, "job": 1, "event": "recovery_finished"} 2024-09-16T08:17:39.928 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:39 smithi028 ceph-mon[64115]: rocksdb: [db/version_set.cc:5047] Creating manifest 35 2024-09-16T08:17:39.928 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:39 smithi028 ceph-mon[64115]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-09-16T08:17:39.929 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:39 smithi028 ceph-mon[64115]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-smithi028/store.db/000028.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-09-16T08:17:39.929 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:39 smithi028 ceph-mon[64115]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x55d70460ae00 2024-09-16T08:17:39.929 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:39 smithi028 ceph-mon[64115]: rocksdb: DB pointer 0x55d70470c000 2024-09-16T08:17:39.929 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:39 smithi028 ceph-mon[64115]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-09-16T08:17:39.929 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:39 smithi028 ceph-mon[64115]: rocksdb: [db/db_impl/db_impl.cc:1111] 2024-09-16T08:17:39.929 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout: ** DB Stats ** 2024-09-16T08:17:39.929 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-16T08:17:39.929 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-09-16T08:17:39.929 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-16T08:17:39.929 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-16T08:17:39.929 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-09-16T08:17:39.930 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-16T08:17:39.930 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-16T08:17:39.930 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout: 2024-09-16T08:17:39.930 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout: ** Compaction Stats [default] ** 2024-09-16T08:17:39.930 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-09-16T08:17:39.931 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-09-16T08:17:39.931 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout: L0 1/0 2.20 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 475.3 0.00 0.00 1 0.005 0 0 0.0 0.0 2024-09-16T08:17:39.931 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout: L6 1/0 7.02 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2024-09-16T08:17:39.931 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout: Sum 2/0 9.22 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 475.3 0.00 0.00 1 0.005 0 0 0.0 0.0 2024-09-16T08:17:39.931 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 475.3 0.00 0.00 1 0.005 0 0 0.0 0.0 2024-09-16T08:17:39.931 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout: 2024-09-16T08:17:39.931 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout: ** Compaction Stats [default] ** 2024-09-16T08:17:39.931 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-09-16T08:17:39.931 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-16T08:17:39.931 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 475.3 0.00 0.00 1 0.005 0 0 0.0 0.0 2024-09-16T08:17:39.931 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout: 2024-09-16T08:17:39.931 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-09-16T08:17:39.932 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout: 2024-09-16T08:17:39.932 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-16T08:17:39.932 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout: Flush(GB): cumulative 0.002, interval 0.002 2024-09-16T08:17:39.932 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-09-16T08:17:39.932 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-09-16T08:17:39.932 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-09-16T08:17:39.932 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout: AddFile(Keys): cumulative 0, interval 0 2024-09-16T08:17:39.932 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout: Cumulative compaction: 0.00 GB write, 194.63 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-16T08:17:39.932 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout: Interval compaction: 0.00 GB write, 194.63 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-16T08:17:39.932 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-09-16T08:17:39.932 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout: Block cache BinnedLRUCache@0x55d7045db610#2 capacity: 512.00 MB usage: 3.98 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1e-05 secs_since: 0 2024-09-16T08:17:39.933 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,1.16 KB,0.000220537%) IndexBlock(1,2.83 KB,0.000539422%) Misc(1,0.00 KB,0%) 2024-09-16T08:17:39.933 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout: 2024-09-16T08:17:39.933 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout: ** File Read Latency Histogram By Level [default] ** 2024-09-16T08:17:39.933 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:39 smithi028 ceph-mon[64115]: starting mon.smithi028 rank 0 at public addrs [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] at bind addrs [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] mon_data /var/lib/ceph/mon/ceph-smithi028 fsid 5622b2fe-7402-11ef-bceb-c7b262605968 2024-09-16T08:17:39.933 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:39 smithi028 ceph-mon[64115]: mon.smithi028@-1(???) e2 preinit fsid 5622b2fe-7402-11ef-bceb-c7b262605968 2024-09-16T08:17:39.933 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:39 smithi028 ceph-mon[64115]: mon.smithi028@-1(???).mds e7 new map 2024-09-16T08:17:39.933 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:39 smithi028 ceph-mon[64115]: mon.smithi028@-1(???).mds e7 print_map 2024-09-16T08:17:39.933 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout: e7 2024-09-16T08:17:39.933 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout: btime 1970-01-01T00:00:00:000000+0000 2024-09-16T08:17:39.933 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-09-16T08:17:39.933 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-09-16T08:17:39.933 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout: legacy client fscid: 1 2024-09-16T08:17:39.933 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout: 2024-09-16T08:17:39.933 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout: Filesystem 'foofs' (1) 2024-09-16T08:17:39.934 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout: fs_name foofs 2024-09-16T08:17:39.934 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout: epoch 5 2024-09-16T08:17:39.934 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout: flags 12 joinable allow_snaps allow_multimds_snaps 2024-09-16T08:17:39.934 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout: created 2024-09-16T08:12:22.241379+0000 2024-09-16T08:17:39.934 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout: modified 2024-09-16T08:12:28.262657+0000 2024-09-16T08:17:39.934 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout: tableserver 0 2024-09-16T08:17:39.934 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout: root 0 2024-09-16T08:17:39.934 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout: session_timeout 60 2024-09-16T08:17:39.934 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout: session_autoclose 300 2024-09-16T08:17:39.934 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout: max_file_size 1099511627776 2024-09-16T08:17:39.934 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout: max_xattr_size 65536 2024-09-16T08:17:39.934 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout: required_client_features {} 2024-09-16T08:17:39.934 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout: last_failure 0 2024-09-16T08:17:39.935 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout: last_failure_osd_epoch 0 2024-09-16T08:17:39.935 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-09-16T08:17:39.935 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout: max_mds 1 2024-09-16T08:17:39.935 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout: in 0 2024-09-16T08:17:39.935 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout: up {0=14480} 2024-09-16T08:17:39.935 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout: failed 2024-09-16T08:17:39.935 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout: damaged 2024-09-16T08:17:39.935 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout: stopped 2024-09-16T08:17:39.935 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout: data_pools [3] 2024-09-16T08:17:39.936 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout: metadata_pool 2 2024-09-16T08:17:39.936 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout: inline_data disabled 2024-09-16T08:17:39.936 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout: balancer 2024-09-16T08:17:39.936 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout: bal_rank_mask -1 2024-09-16T08:17:39.936 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout: standby_count_wanted 1 2024-09-16T08:17:39.936 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout: qdb_cluster leader: 0 members: 2024-09-16T08:17:39.936 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout: [mds.foofs.smithi028.srtgev{0:14480} state up:active seq 2 join_fscid=1 addr [v2:172.21.15.28:6834/800761142,v1:172.21.15.28:6835/800761142] compat {c=[1],r=[1],i=[7ff]}] 2024-09-16T08:17:39.936 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout: 2024-09-16T08:17:39.936 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout: 2024-09-16T08:17:39.936 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout: Standby daemons: 2024-09-16T08:17:39.936 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout: 2024-09-16T08:17:39.936 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout: [mds.foofs.smithi144.oozgsi{-1:24297} state up:standby seq 1 join_fscid=1 addr [v2:172.21.15.144:6832/2267142930,v1:172.21.15.144:6833/2267142930] compat {c=[1],r=[1],i=[7ff]}] 2024-09-16T08:17:39.936 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:39 smithi028 ceph-mon[64115]: mon.smithi028@-1(???).osd e71 crush map has features 3314933000854323200, adjusting msgr requires 2024-09-16T08:17:39.936 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:39 smithi028 ceph-mon[64115]: mon.smithi028@-1(???).osd e71 crush map has features 432629239337189376, adjusting msgr requires 2024-09-16T08:17:39.937 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:39 smithi028 ceph-mon[64115]: mon.smithi028@-1(???).osd e71 crush map has features 432629239337189376, adjusting msgr requires 2024-09-16T08:17:39.937 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:39 smithi028 ceph-mon[64115]: mon.smithi028@-1(???).osd e71 crush map has features 432629239337189376, adjusting msgr requires 2024-09-16T08:17:39.937 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:39 smithi028 ceph-mon[64115]: mon.smithi028@-1(???).paxosservice(auth 1..29) refresh upgraded, format 0 -> 3 2024-09-16T08:17:40.035 INFO:teuthology.orchestra.run.smithi028.stdout: 5 132331 10.81 MB/sec execute 158 sec latency 57.822 ms 2024-09-16T08:17:40.163 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:39 smithi028 bash[64097]: 14efcedc0088717f01c964e67e71441d1547d4705d5236fd6e00c0beff69d513 2024-09-16T08:17:40.163 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:39 smithi028 systemd[1]: Started Ceph mon.smithi028 for 5622b2fe-7402-11ef-bceb-c7b262605968. 2024-09-16T08:17:40.163 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:40 smithi028 ceph-mon[64115]: pgmap v26: 97 pgs: 97 active+clean; 329 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 8.1 KiB/s rd, 4.2 MiB/s wr, 376 op/s 2024-09-16T08:17:40.163 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:40 smithi028 ceph-mon[64115]: pgmap v27: 97 pgs: 97 active+clean; 336 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 5.8 KiB/s rd, 5.0 MiB/s wr, 451 op/s 2024-09-16T08:17:40.163 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:40 smithi028 ceph-mon[64115]: mon.smithi028 calling monitor election 2024-09-16T08:17:40.163 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:40 smithi028 ceph-mon[64115]: mon.smithi028 is new leader, mons smithi028,smithi144 in quorum (ranks 0,1) 2024-09-16T08:17:40.164 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:40 smithi028 ceph-mon[64115]: monmap epoch 2 2024-09-16T08:17:40.164 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:40 smithi028 ceph-mon[64115]: fsid 5622b2fe-7402-11ef-bceb-c7b262605968 2024-09-16T08:17:40.164 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:40 smithi028 ceph-mon[64115]: last_changed 2024-09-16T08:07:20.025003+0000 2024-09-16T08:17:40.164 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:40 smithi028 ceph-mon[64115]: created 2024-09-16T08:05:16.161732+0000 2024-09-16T08:17:40.164 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:40 smithi028 ceph-mon[64115]: min_mon_release 17 (quincy) 2024-09-16T08:17:40.164 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:40 smithi028 ceph-mon[64115]: election_strategy: 1 2024-09-16T08:17:40.165 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:40 smithi028 ceph-mon[64115]: 0: [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] mon.smithi028 2024-09-16T08:17:40.165 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:40 smithi028 ceph-mon[64115]: 1: [v2:172.21.15.144:3300/0,v1:172.21.15.144:6789/0] mon.smithi144 2024-09-16T08:17:40.165 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:40 smithi028 ceph-mon[64115]: fsmap foofs:1 {0=foofs.smithi028.srtgev=up:active} 1 up:standby 2024-09-16T08:17:40.165 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:40 smithi028 ceph-mon[64115]: osdmap e71: 8 total, 8 up, 8 in 2024-09-16T08:17:40.165 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:40 smithi028 ceph-mon[64115]: mgrmap e35: smithi028.azvyzy(active, since 50s), standbys: smithi144.utzszj 2024-09-16T08:17:40.165 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:40 smithi028 ceph-mon[64115]: overall HEALTH_OK 2024-09-16T08:17:40.166 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:40 smithi028 ceph-mon[64115]: mgrmap e36: smithi028.azvyzy(active, since 50s), standbys: smithi144.utzszj 2024-09-16T08:17:40.166 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:40 smithi028 ceph-mon[64115]: from='mgr.14768 ' entity='' 2024-09-16T08:17:40.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:17:40 smithi144 ceph-mon[25668]: pgmap v26: 97 pgs: 97 active+clean; 329 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 8.1 KiB/s rd, 4.2 MiB/s wr, 376 op/s 2024-09-16T08:17:40.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:17:40 smithi144 ceph-mon[25668]: pgmap v27: 97 pgs: 97 active+clean; 336 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 5.8 KiB/s rd, 5.0 MiB/s wr, 451 op/s 2024-09-16T08:17:40.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:17:40 smithi144 ceph-mon[25668]: mon.smithi028 calling monitor election 2024-09-16T08:17:40.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:17:40 smithi144 ceph-mon[25668]: mon.smithi028 is new leader, mons smithi028,smithi144 in quorum (ranks 0,1) 2024-09-16T08:17:40.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:17:40 smithi144 ceph-mon[25668]: monmap epoch 2 2024-09-16T08:17:40.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:17:40 smithi144 ceph-mon[25668]: fsid 5622b2fe-7402-11ef-bceb-c7b262605968 2024-09-16T08:17:40.583 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:17:40 smithi144 ceph-mon[25668]: last_changed 2024-09-16T08:07:20.025003+0000 2024-09-16T08:17:40.583 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:17:40 smithi144 ceph-mon[25668]: created 2024-09-16T08:05:16.161732+0000 2024-09-16T08:17:40.583 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:17:40 smithi144 ceph-mon[25668]: min_mon_release 17 (quincy) 2024-09-16T08:17:40.583 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:17:40 smithi144 ceph-mon[25668]: election_strategy: 1 2024-09-16T08:17:40.583 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:17:40 smithi144 ceph-mon[25668]: 0: [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] mon.smithi028 2024-09-16T08:17:40.583 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:17:40 smithi144 ceph-mon[25668]: 1: [v2:172.21.15.144:3300/0,v1:172.21.15.144:6789/0] mon.smithi144 2024-09-16T08:17:40.583 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:17:40 smithi144 ceph-mon[25668]: fsmap foofs:1 {0=foofs.smithi028.srtgev=up:active} 1 up:standby 2024-09-16T08:17:40.583 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:17:40 smithi144 ceph-mon[25668]: osdmap e71: 8 total, 8 up, 8 in 2024-09-16T08:17:40.583 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:17:40 smithi144 ceph-mon[25668]: mgrmap e35: smithi028.azvyzy(active, since 50s), standbys: smithi144.utzszj 2024-09-16T08:17:40.583 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:17:40 smithi144 ceph-mon[25668]: overall HEALTH_OK 2024-09-16T08:17:40.583 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:17:40 smithi144 ceph-mon[25668]: mgrmap e36: smithi028.azvyzy(active, since 50s), standbys: smithi144.utzszj 2024-09-16T08:17:40.583 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:17:40 smithi144 ceph-mon[25668]: from='mgr.14768 ' entity='' 2024-09-16T08:17:41.035 INFO:teuthology.orchestra.run.smithi028.stdout: 5 132664 10.79 MB/sec execute 159 sec latency 66.197 ms 2024-09-16T08:17:42.035 INFO:teuthology.orchestra.run.smithi028.stdout: 5 132996 10.75 MB/sec execute 160 sec latency 62.405 ms 2024-09-16T08:17:42.197 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:42 smithi028 ceph-mon[64115]: mgrmap e37: smithi028.azvyzy(active, since 51s), standbys: smithi144.utzszj 2024-09-16T08:17:42.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:17:42 smithi144 ceph-mon[25668]: mgrmap e37: smithi028.azvyzy(active, since 51s), standbys: smithi144.utzszj 2024-09-16T08:17:43.035 INFO:teuthology.orchestra.run.smithi028.stdout: 5 133320 10.75 MB/sec execute 161 sec latency 50.843 ms 2024-09-16T08:17:44.035 INFO:teuthology.orchestra.run.smithi028.stdout: 5 133597 10.73 MB/sec execute 162 sec latency 70.606 ms 2024-09-16T08:17:45.035 INFO:teuthology.orchestra.run.smithi028.stdout: 5 133944 10.73 MB/sec execute 163 sec latency 77.741 ms 2024-09-16T08:17:46.035 INFO:teuthology.orchestra.run.smithi028.stdout: 5 134212 10.71 MB/sec execute 164 sec latency 58.878 ms 2024-09-16T08:17:46.668 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:46 smithi028 ceph-mon[64115]: from='mgr.? 172.21.15.144:0/141259300' entity='mgr.smithi144.utzszj' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi144.utzszj/crt"}]: dispatch 2024-09-16T08:17:46.668 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:46 smithi028 ceph-mon[64115]: from='mgr.? 172.21.15.144:0/141259300' entity='mgr.smithi144.utzszj' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-16T08:17:46.668 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:46 smithi028 ceph-mon[64115]: from='mgr.? 172.21.15.144:0/141259300' entity='mgr.smithi144.utzszj' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi144.utzszj/key"}]: dispatch 2024-09-16T08:17:46.668 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:46 smithi028 ceph-mon[64115]: from='mgr.? 172.21.15.144:0/141259300' entity='mgr.smithi144.utzszj' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-16T08:17:46.668 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:46 smithi028 ceph-mon[64115]: Standby manager daemon smithi144.utzszj restarted 2024-09-16T08:17:46.668 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:46 smithi028 ceph-mon[64115]: Standby manager daemon smithi144.utzszj started 2024-09-16T08:17:46.831 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:17:46 smithi144 ceph-mon[25668]: from='mgr.? 172.21.15.144:0/141259300' entity='mgr.smithi144.utzszj' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi144.utzszj/crt"}]: dispatch 2024-09-16T08:17:46.832 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:17:46 smithi144 ceph-mon[25668]: from='mgr.? 172.21.15.144:0/141259300' entity='mgr.smithi144.utzszj' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-16T08:17:46.832 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:17:46 smithi144 ceph-mon[25668]: from='mgr.? 172.21.15.144:0/141259300' entity='mgr.smithi144.utzszj' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi144.utzszj/key"}]: dispatch 2024-09-16T08:17:46.832 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:17:46 smithi144 ceph-mon[25668]: from='mgr.? 172.21.15.144:0/141259300' entity='mgr.smithi144.utzszj' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-16T08:17:46.832 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:17:46 smithi144 ceph-mon[25668]: Standby manager daemon smithi144.utzszj restarted 2024-09-16T08:17:46.832 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:17:46 smithi144 ceph-mon[25668]: Standby manager daemon smithi144.utzszj started 2024-09-16T08:17:47.035 INFO:teuthology.orchestra.run.smithi028.stdout: 5 134449 10.70 MB/sec execute 165 sec latency 62.863 ms 2024-09-16T08:17:47.697 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:47 smithi028 ceph-mon[64115]: mgrmap e38: smithi028.azvyzy(active, since 56s), standbys: smithi144.utzszj 2024-09-16T08:17:47.698 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:47 smithi028 ceph-mon[64115]: Active manager daemon smithi028.azvyzy restarted 2024-09-16T08:17:47.698 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:47 smithi028 ceph-mon[64115]: Activating manager daemon smithi028.azvyzy 2024-09-16T08:17:47.698 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:47 smithi028 ceph-mon[64115]: osdmap e72: 8 total, 8 up, 8 in 2024-09-16T08:17:47.698 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:47 smithi028 ceph-mon[64115]: mgrmap e39: smithi028.azvyzy(active, starting, since 0.0303081s), standbys: smithi144.utzszj 2024-09-16T08:17:47.698 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:47 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "mon metadata", "id": "smithi028"} : dispatch 2024-09-16T08:17:47.698 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:47 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "mon metadata", "id": "smithi144"} : dispatch 2024-09-16T08:17:47.698 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:47 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "mds metadata", "who": "foofs.smithi028.srtgev"} : dispatch 2024-09-16T08:17:47.698 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:47 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "mds metadata", "who": "foofs.smithi144.oozgsi"} : dispatch 2024-09-16T08:17:47.698 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:47 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "mgr metadata", "who": "smithi028.azvyzy", "id": "smithi028.azvyzy"} : dispatch 2024-09-16T08:17:47.698 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:47 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "mgr metadata", "who": "smithi144.utzszj", "id": "smithi144.utzszj"} : dispatch 2024-09-16T08:17:47.698 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:47 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "osd metadata", "id": 0} : dispatch 2024-09-16T08:17:47.698 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:47 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "osd metadata", "id": 1} : dispatch 2024-09-16T08:17:47.698 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:47 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "osd metadata", "id": 2} : dispatch 2024-09-16T08:17:47.699 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:47 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "osd metadata", "id": 3} : dispatch 2024-09-16T08:17:47.699 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:47 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "osd metadata", "id": 4} : dispatch 2024-09-16T08:17:47.699 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:47 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "osd metadata", "id": 5} : dispatch 2024-09-16T08:17:47.699 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:47 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "osd metadata", "id": 6} : dispatch 2024-09-16T08:17:47.699 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:47 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "osd metadata", "id": 7} : dispatch 2024-09-16T08:17:47.699 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:47 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "mds metadata"} : dispatch 2024-09-16T08:17:47.699 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:47 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "osd metadata"} : dispatch 2024-09-16T08:17:47.699 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:47 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "mon metadata"} : dispatch 2024-09-16T08:17:47.699 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:47 smithi028 ceph-mon[64115]: Manager daemon smithi028.azvyzy is now available 2024-09-16T08:17:47.699 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:47 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-16T08:17:47.699 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:47 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-16T08:17:47.699 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:47 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "orch prometheus get-credentials"} : dispatch 2024-09-16T08:17:47.699 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:47 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi028.azvyzy/mirror_snapshot_schedule"} : dispatch 2024-09-16T08:17:47.699 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:47 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi028.azvyzy/trash_purge_schedule"} : dispatch 2024-09-16T08:17:47.832 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:17:47 smithi144 ceph-mon[25668]: mgrmap e38: smithi028.azvyzy(active, since 56s), standbys: smithi144.utzszj 2024-09-16T08:17:47.832 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:17:47 smithi144 ceph-mon[25668]: Active manager daemon smithi028.azvyzy restarted 2024-09-16T08:17:47.832 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:17:47 smithi144 ceph-mon[25668]: Activating manager daemon smithi028.azvyzy 2024-09-16T08:17:47.832 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:17:47 smithi144 ceph-mon[25668]: osdmap e72: 8 total, 8 up, 8 in 2024-09-16T08:17:47.832 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:17:47 smithi144 ceph-mon[25668]: mgrmap e39: smithi028.azvyzy(active, starting, since 0.0303081s), standbys: smithi144.utzszj 2024-09-16T08:17:47.833 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:17:47 smithi144 ceph-mon[25668]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "mon metadata", "id": "smithi028"} : dispatch 2024-09-16T08:17:47.833 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:17:47 smithi144 ceph-mon[25668]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "mon metadata", "id": "smithi144"} : dispatch 2024-09-16T08:17:47.833 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:17:47 smithi144 ceph-mon[25668]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "mds metadata", "who": "foofs.smithi028.srtgev"} : dispatch 2024-09-16T08:17:47.833 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:17:47 smithi144 ceph-mon[25668]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "mds metadata", "who": "foofs.smithi144.oozgsi"} : dispatch 2024-09-16T08:17:47.833 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:17:47 smithi144 ceph-mon[25668]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "mgr metadata", "who": "smithi028.azvyzy", "id": "smithi028.azvyzy"} : dispatch 2024-09-16T08:17:47.833 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:17:47 smithi144 ceph-mon[25668]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "mgr metadata", "who": "smithi144.utzszj", "id": "smithi144.utzszj"} : dispatch 2024-09-16T08:17:47.833 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:17:47 smithi144 ceph-mon[25668]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "osd metadata", "id": 0} : dispatch 2024-09-16T08:17:47.833 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:17:47 smithi144 ceph-mon[25668]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "osd metadata", "id": 1} : dispatch 2024-09-16T08:17:47.833 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:17:47 smithi144 ceph-mon[25668]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "osd metadata", "id": 2} : dispatch 2024-09-16T08:17:47.833 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:17:47 smithi144 ceph-mon[25668]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "osd metadata", "id": 3} : dispatch 2024-09-16T08:17:47.833 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:17:47 smithi144 ceph-mon[25668]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "osd metadata", "id": 4} : dispatch 2024-09-16T08:17:47.833 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:17:47 smithi144 ceph-mon[25668]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "osd metadata", "id": 5} : dispatch 2024-09-16T08:17:47.834 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:17:47 smithi144 ceph-mon[25668]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "osd metadata", "id": 6} : dispatch 2024-09-16T08:17:47.834 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:17:47 smithi144 ceph-mon[25668]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "osd metadata", "id": 7} : dispatch 2024-09-16T08:17:47.834 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:17:47 smithi144 ceph-mon[25668]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "mds metadata"} : dispatch 2024-09-16T08:17:47.834 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:17:47 smithi144 ceph-mon[25668]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "osd metadata"} : dispatch 2024-09-16T08:17:47.834 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:17:47 smithi144 ceph-mon[25668]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "mon metadata"} : dispatch 2024-09-16T08:17:47.834 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:17:47 smithi144 ceph-mon[25668]: Manager daemon smithi028.azvyzy is now available 2024-09-16T08:17:47.834 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:17:47 smithi144 ceph-mon[25668]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-16T08:17:47.834 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:17:47 smithi144 ceph-mon[25668]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-16T08:17:47.834 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:17:47 smithi144 ceph-mon[25668]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "orch prometheus get-credentials"} : dispatch 2024-09-16T08:17:47.834 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:17:47 smithi144 ceph-mon[25668]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi028.azvyzy/mirror_snapshot_schedule"} : dispatch 2024-09-16T08:17:47.834 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:17:47 smithi144 ceph-mon[25668]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi028.azvyzy/trash_purge_schedule"} : dispatch 2024-09-16T08:17:48.035 INFO:teuthology.orchestra.run.smithi028.stdout: 5 134690 10.71 MB/sec execute 166 sec latency 62.775 ms 2024-09-16T08:17:48.831 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:17:48 smithi144 ceph-mon[25668]: mgrmap e40: smithi028.azvyzy(active, since 1.03717s), standbys: smithi144.utzszj 2024-09-16T08:17:48.831 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:17:48 smithi144 ceph-mon[25668]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-16T08:17:48.832 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:17:48 smithi144 ceph-mon[25668]: pgmap v3: 97 pgs: 97 active+clean; 328 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-09-16T08:17:48.992 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:48 smithi028 ceph-mon[64115]: mgrmap e40: smithi028.azvyzy(active, since 1.03717s), standbys: smithi144.utzszj 2024-09-16T08:17:48.993 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:48 smithi028 ceph-mon[64115]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-16T08:17:48.993 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:48 smithi028 ceph-mon[64115]: pgmap v3: 97 pgs: 97 active+clean; 328 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-09-16T08:17:49.035 INFO:teuthology.orchestra.run.smithi028.stdout: 5 134918 10.68 MB/sec execute 167 sec latency 67.768 ms 2024-09-16T08:17:50.013 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:49 smithi028 ceph-mon[64115]: [16/Sep/2024:08:17:48] ENGINE Bus STARTING 2024-09-16T08:17:50.013 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:49 smithi028 ceph-mon[64115]: [16/Sep/2024:08:17:48] ENGINE Serving on http://172.21.15.28:8765 2024-09-16T08:17:50.013 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:49 smithi028 ceph-mon[64115]: [16/Sep/2024:08:17:48] ENGINE Serving on https://172.21.15.28:7150 2024-09-16T08:17:50.013 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:49 smithi028 ceph-mon[64115]: [16/Sep/2024:08:17:48] ENGINE Bus STARTED 2024-09-16T08:17:50.013 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:49 smithi028 ceph-mon[64115]: [16/Sep/2024:08:17:48] ENGINE Error in HTTPServer.serve 2024-09-16T08:17:50.013 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout: Traceback (most recent call last): 2024-09-16T08:17:50.013 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-16T08:17:50.013 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout: self._connections.run(self.expiration_interval) 2024-09-16T08:17:50.013 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-16T08:17:50.014 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout: self._run(expiration_interval) 2024-09-16T08:17:50.014 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-16T08:17:50.014 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-16T08:17:50.014 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-16T08:17:50.014 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-16T08:17:50.014 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-16T08:17:50.014 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout: s = self.context.wrap_socket( 2024-09-16T08:17:50.014 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-16T08:17:50.014 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout: return self.sslsocket_class._create( 2024-09-16T08:17:50.014 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-16T08:17:50.014 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout: self.do_handshake() 2024-09-16T08:17:50.014 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-16T08:17:50.014 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout: self._sslobj.do_handshake() 2024-09-16T08:17:50.015 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-16T08:17:50.015 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:49 smithi028 ceph-mon[64115]: pgmap v4: 97 pgs: 97 active+clean; 328 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-09-16T08:17:50.015 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:49 smithi028 ceph-mon[64115]: mgrmap e41: smithi028.azvyzy(active, since 2s), standbys: smithi144.utzszj 2024-09-16T08:17:50.035 INFO:teuthology.orchestra.run.smithi028.stdout: 5 135211 10.65 MB/sec execute 168 sec latency 54.139 ms 2024-09-16T08:17:50.046 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:17:49 smithi144 ceph-mon[25668]: [16/Sep/2024:08:17:48] ENGINE Bus STARTING 2024-09-16T08:17:50.047 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:17:49 smithi144 ceph-mon[25668]: [16/Sep/2024:08:17:48] ENGINE Serving on http://172.21.15.28:8765 2024-09-16T08:17:50.047 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:17:49 smithi144 ceph-mon[25668]: [16/Sep/2024:08:17:48] ENGINE Serving on https://172.21.15.28:7150 2024-09-16T08:17:50.047 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:17:49 smithi144 ceph-mon[25668]: [16/Sep/2024:08:17:48] ENGINE Bus STARTED 2024-09-16T08:17:50.047 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:17:49 smithi144 ceph-mon[25668]: [16/Sep/2024:08:17:48] ENGINE Error in HTTPServer.serve 2024-09-16T08:17:50.047 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: Traceback (most recent call last): 2024-09-16T08:17:50.047 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-16T08:17:50.047 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: self._connections.run(self.expiration_interval) 2024-09-16T08:17:50.047 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-16T08:17:50.047 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: self._run(expiration_interval) 2024-09-16T08:17:50.047 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-16T08:17:50.047 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-16T08:17:50.047 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-16T08:17:50.047 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-16T08:17:50.047 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-16T08:17:50.048 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: s = self.context.wrap_socket( 2024-09-16T08:17:50.048 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-16T08:17:50.048 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: return self.sslsocket_class._create( 2024-09-16T08:17:50.048 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-16T08:17:50.048 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: self.do_handshake() 2024-09-16T08:17:50.048 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-16T08:17:50.048 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: self._sslobj.do_handshake() 2024-09-16T08:17:50.048 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-16T08:17:50.048 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:17:49 smithi144 ceph-mon[25668]: pgmap v4: 97 pgs: 97 active+clean; 328 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-09-16T08:17:50.048 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:17:49 smithi144 ceph-mon[25668]: mgrmap e41: smithi028.azvyzy(active, since 2s), standbys: smithi144.utzszj 2024-09-16T08:17:51.035 INFO:teuthology.orchestra.run.smithi028.stdout: 5 135459 10.62 MB/sec execute 169 sec latency 64.006 ms 2024-09-16T08:17:51.796 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:51 smithi028 ceph-mon[64115]: pgmap v5: 97 pgs: 97 active+clean; 328 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-09-16T08:17:51.796 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:51 smithi028 ceph-mon[64115]: mgrmap e42: smithi028.azvyzy(active, since 4s), standbys: smithi144.utzszj 2024-09-16T08:17:51.796 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:51 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:17:51.796 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:51 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:17:52.035 INFO:teuthology.orchestra.run.smithi028.stdout: 5 135686 10.60 MB/sec execute 170 sec latency 63.921 ms 2024-09-16T08:17:52.081 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:17:51 smithi144 ceph-mon[25668]: pgmap v5: 97 pgs: 97 active+clean; 328 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-09-16T08:17:52.082 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:17:51 smithi144 ceph-mon[25668]: mgrmap e42: smithi028.azvyzy(active, since 4s), standbys: smithi144.utzszj 2024-09-16T08:17:52.082 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:17:51 smithi144 ceph-mon[25668]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:17:52.082 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:17:51 smithi144 ceph-mon[25668]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:17:53.035 INFO:teuthology.orchestra.run.smithi028.stdout: 5 135935 10.55 MB/sec execute 171 sec latency 61.937 ms 2024-09-16T08:17:53.320 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:53 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:17:53.320 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:53 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:17:53.320 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:53 smithi028 ceph-mon[64115]: pgmap v6: 97 pgs: 97 active+clean; 328 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-09-16T08:17:53.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:17:53 smithi144 ceph-mon[25668]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:17:53.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:17:53 smithi144 ceph-mon[25668]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:17:53.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:17:53 smithi144 ceph-mon[25668]: pgmap v6: 97 pgs: 97 active+clean; 328 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-09-16T08:17:54.035 INFO:teuthology.orchestra.run.smithi028.stdout: 5 136229 10.54 MB/sec execute 172 sec latency 63.781 ms 2024-09-16T08:17:55.035 INFO:teuthology.orchestra.run.smithi028.stdout: 5 136480 10.49 MB/sec execute 173 sec latency 77.042 ms 2024-09-16T08:17:55.175 INFO:teuthology.orchestra.run.smithi028.stdout:true 2024-09-16T08:17:55.866 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:55 smithi028 ceph-mon[64115]: pgmap v7: 97 pgs: 97 active+clean; 326 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 31 KiB/s rd, 2.8 MiB/s wr, 293 op/s 2024-09-16T08:17:55.866 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:55 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:17:55.866 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:55 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:17:56.035 INFO:teuthology.orchestra.run.smithi028.stdout: 5 136755 10.48 MB/sec execute 174 sec latency 66.971 ms 2024-09-16T08:17:56.081 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:17:55 smithi144 ceph-mon[25668]: pgmap v7: 97 pgs: 97 active+clean; 326 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 31 KiB/s rd, 2.8 MiB/s wr, 293 op/s 2024-09-16T08:17:56.081 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:17:55 smithi144 ceph-mon[25668]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:17:56.081 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:17:55 smithi144 ceph-mon[25668]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:17:56.216 INFO:teuthology.orchestra.run.smithi028.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-16T08:17:56.216 INFO:teuthology.orchestra.run.smithi028.stdout:alertmanager.smithi028 smithi028 *:9093,9094 running (2m) 0s ago 11m 19.7M - 0.27.0 11f11916f8cd a14fcb4cb2b6 2024-09-16T08:17:56.216 INFO:teuthology.orchestra.run.smithi028.stdout:crash.smithi028 smithi028 running (11m) 0s ago 11m 6405k - 17.2.0 e1d6a67b021e 69e795c000fd 2024-09-16T08:17:56.216 INFO:teuthology.orchestra.run.smithi028.stdout:crash.smithi144 smithi144 running (10m) 4s ago 10m 6400k - 17.2.0 e1d6a67b021e bbe54b51649c 2024-09-16T08:17:56.216 INFO:teuthology.orchestra.run.smithi028.stdout:grafana.smithi028 smithi028 *:3000 running (2m) 0s ago 11m 43.8M - dad864ee21e9 b55ba8b944a1 2024-09-16T08:17:56.216 INFO:teuthology.orchestra.run.smithi028.stdout:mds.foofs.smithi028.srtgev smithi028 running (5m) 0s ago 5m 37.1M - 17.2.0 e1d6a67b021e 250e7a7f754c 2024-09-16T08:17:56.216 INFO:teuthology.orchestra.run.smithi028.stdout:mds.foofs.smithi144.oozgsi smithi144 running (5m) 4s ago 5m 16.3M - 17.2.0 e1d6a67b021e ecf4c2bb0fdc 2024-09-16T08:17:56.217 INFO:teuthology.orchestra.run.smithi028.stdout:mgr.smithi028.azvyzy smithi028 *:8443,9283,8765 running (83s) 0s ago 12m 488M - 19.3.0-4917-g8293d73f 425e316ba8eb e01d4382d03f 2024-09-16T08:17:56.217 INFO:teuthology.orchestra.run.smithi028.stdout:mgr.smithi144.utzszj smithi144 *:8443,9283,8765 running (34s) 4s ago 10m 451M - 19.3.0-4917-g8293d73f 425e316ba8eb 71f7f802bffb 2024-09-16T08:17:56.217 INFO:teuthology.orchestra.run.smithi028.stdout:mon.smithi028 smithi028 running (16s) 0s ago 12m 31.9M 2048M 19.3.0-4917-g8293d73f 425e316ba8eb 14efcedc0088 2024-09-16T08:17:56.217 INFO:teuthology.orchestra.run.smithi028.stdout:mon.smithi144 smithi144 running (10m) 4s ago 10m 55.2M 2048M 17.2.0 e1d6a67b021e 1c80e3ff3507 2024-09-16T08:17:56.217 INFO:teuthology.orchestra.run.smithi028.stdout:nfs.foo.0.0.smithi028.dkttgj smithi028 *:2049 running (5m) 0s ago 5m 274M - 4.0 e1d6a67b021e b5dccc45a6ed 2024-09-16T08:17:56.217 INFO:teuthology.orchestra.run.smithi028.stdout:nfs.foo.1.0.smithi144.epipkg smithi144 *:2049 running (5m) 4s ago 5m 86.4M - 4.0 e1d6a67b021e faa89e61f50e 2024-09-16T08:17:56.217 INFO:teuthology.orchestra.run.smithi028.stdout:node-exporter.smithi028 smithi028 *:9100 running (2m) 0s ago 11m 9977k - 1.7.0 72c9c2088986 9e2b8e8a63a9 2024-09-16T08:17:56.218 INFO:teuthology.orchestra.run.smithi028.stdout:node-exporter.smithi144 smithi144 *:9100 running (92s) 4s ago 10m 10.2M - 1.7.0 72c9c2088986 78834c5e992f 2024-09-16T08:17:56.218 INFO:teuthology.orchestra.run.smithi028.stdout:osd.0 smithi028 running (9m) 0s ago 9m 485M 1937M 17.2.0 e1d6a67b021e 8184fbc3871f 2024-09-16T08:17:56.218 INFO:teuthology.orchestra.run.smithi028.stdout:osd.1 smithi028 running (9m) 0s ago 9m 408M 1937M 17.2.0 e1d6a67b021e 6911d5c68d54 2024-09-16T08:17:56.218 INFO:teuthology.orchestra.run.smithi028.stdout:osd.2 smithi028 running (8m) 0s ago 8m 299M 1937M 17.2.0 e1d6a67b021e 90b27d8375a2 2024-09-16T08:17:56.218 INFO:teuthology.orchestra.run.smithi028.stdout:osd.3 smithi028 running (8m) 0s ago 8m 448M 1937M 17.2.0 e1d6a67b021e dac477d8b04f 2024-09-16T08:17:56.218 INFO:teuthology.orchestra.run.smithi028.stdout:osd.4 smithi144 running (8m) 4s ago 8m 473M 2704M 17.2.0 e1d6a67b021e 86719c60975c 2024-09-16T08:17:56.218 INFO:teuthology.orchestra.run.smithi028.stdout:osd.5 smithi144 running (7m) 4s ago 7m 512M 2704M 17.2.0 e1d6a67b021e 70452c145c60 2024-09-16T08:17:56.218 INFO:teuthology.orchestra.run.smithi028.stdout:osd.6 smithi144 running (7m) 4s ago 7m 425M 2704M 17.2.0 e1d6a67b021e 7f9981be792f 2024-09-16T08:17:56.219 INFO:teuthology.orchestra.run.smithi028.stdout:osd.7 smithi144 running (6m) 4s ago 6m 339M 2704M 17.2.0 e1d6a67b021e dada74ddbe4c 2024-09-16T08:17:56.219 INFO:teuthology.orchestra.run.smithi028.stdout:prometheus.smithi028 smithi028 *:9095 running (41s) 0s ago 10m 49.5M - 2.51.0 1d3b7f56885b 42667809df2b 2024-09-16T08:17:56.851 INFO:teuthology.orchestra.run.smithi028.stdout:{ 2024-09-16T08:17:56.851 INFO:teuthology.orchestra.run.smithi028.stdout: "mon": { 2024-09-16T08:17:56.851 INFO:teuthology.orchestra.run.smithi028.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 1, 2024-09-16T08:17:56.852 INFO:teuthology.orchestra.run.smithi028.stdout: "ceph version 19.3.0-4917-g8293d73f (8293d73f8690540e843a81caec373f9cc29cf705) squid (dev)": 1 2024-09-16T08:17:56.852 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-09-16T08:17:56.852 INFO:teuthology.orchestra.run.smithi028.stdout: "mgr": { 2024-09-16T08:17:56.852 INFO:teuthology.orchestra.run.smithi028.stdout: "ceph version 19.3.0-4917-g8293d73f (8293d73f8690540e843a81caec373f9cc29cf705) squid (dev)": 2 2024-09-16T08:17:56.852 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-09-16T08:17:56.852 INFO:teuthology.orchestra.run.smithi028.stdout: "osd": { 2024-09-16T08:17:56.852 INFO:teuthology.orchestra.run.smithi028.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-09-16T08:17:56.852 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-09-16T08:17:56.852 INFO:teuthology.orchestra.run.smithi028.stdout: "mds": { 2024-09-16T08:17:56.852 INFO:teuthology.orchestra.run.smithi028.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-16T08:17:56.852 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-09-16T08:17:56.852 INFO:teuthology.orchestra.run.smithi028.stdout: "overall": { 2024-09-16T08:17:56.852 INFO:teuthology.orchestra.run.smithi028.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 11, 2024-09-16T08:17:56.853 INFO:teuthology.orchestra.run.smithi028.stdout: "ceph version 19.3.0-4917-g8293d73f (8293d73f8690540e843a81caec373f9cc29cf705) squid (dev)": 3 2024-09-16T08:17:56.853 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-09-16T08:17:56.853 INFO:teuthology.orchestra.run.smithi028.stdout:} 2024-09-16T08:17:56.912 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:56 smithi028 ceph-mon[64115]: from='client.34132 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T08:17:56.912 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:56 smithi028 ceph-mon[64115]: from='client.34136 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T08:17:56.912 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:56 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:17:56.912 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:56 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:17:56.912 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:56 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:17:56.912 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:56 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:17:56.912 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:56 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-16T08:17:56.913 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:56 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-16T08:17:56.913 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:56 smithi028 ceph-mon[64115]: from='client.? 172.21.15.28:0/3470336459' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-16T08:17:56.913 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:56 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:17:57.035 INFO:teuthology.orchestra.run.smithi028.stdout: 5 136969 10.45 MB/sec execute 175 sec latency 59.889 ms 2024-09-16T08:17:57.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:17:56 smithi144 ceph-mon[25668]: from='client.34132 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T08:17:57.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:17:56 smithi144 ceph-mon[25668]: from='client.34136 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T08:17:57.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:17:56 smithi144 ceph-mon[25668]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:17:57.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:17:56 smithi144 ceph-mon[25668]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:17:57.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:17:56 smithi144 ceph-mon[25668]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:17:57.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:17:56 smithi144 ceph-mon[25668]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:17:57.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:17:56 smithi144 ceph-mon[25668]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-16T08:17:57.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:17:56 smithi144 ceph-mon[25668]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-16T08:17:57.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:17:56 smithi144 ceph-mon[25668]: from='client.? 172.21.15.28:0/3470336459' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-16T08:17:57.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:17:56 smithi144 ceph-mon[25668]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:17:57.387 INFO:teuthology.orchestra.run.smithi028.stdout:{ 2024-09-16T08:17:57.387 INFO:teuthology.orchestra.run.smithi028.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:f7ad55f9ccd17ba66aadb19b4d291e67b60ea24e5d8a0c0e92941fd59c81e4ce", 2024-09-16T08:17:57.387 INFO:teuthology.orchestra.run.smithi028.stdout: "in_progress": true, 2024-09-16T08:17:57.387 INFO:teuthology.orchestra.run.smithi028.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-16T08:17:57.387 INFO:teuthology.orchestra.run.smithi028.stdout: "services_complete": [ 2024-09-16T08:17:57.387 INFO:teuthology.orchestra.run.smithi028.stdout: "mgr" 2024-09-16T08:17:57.387 INFO:teuthology.orchestra.run.smithi028.stdout: ], 2024-09-16T08:17:57.387 INFO:teuthology.orchestra.run.smithi028.stdout: "progress": "3/23 daemons upgraded", 2024-09-16T08:17:57.387 INFO:teuthology.orchestra.run.smithi028.stdout: "message": "", 2024-09-16T08:17:57.387 INFO:teuthology.orchestra.run.smithi028.stdout: "is_paused": false 2024-09-16T08:17:57.387 INFO:teuthology.orchestra.run.smithi028.stdout:} 2024-09-16T08:17:57.931 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:57 smithi028 ceph-mon[64115]: from='client.24655 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T08:17:57.931 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:57 smithi028 ceph-mon[64115]: Updating smithi028:/etc/ceph/ceph.conf 2024-09-16T08:17:57.932 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:57 smithi028 ceph-mon[64115]: Updating smithi144:/etc/ceph/ceph.conf 2024-09-16T08:17:57.932 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:57 smithi028 ceph-mon[64115]: Updating smithi144:/var/lib/ceph/5622b2fe-7402-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-16T08:17:57.932 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:57 smithi028 ceph-mon[64115]: Updating smithi028:/var/lib/ceph/5622b2fe-7402-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-16T08:17:57.932 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:57 smithi028 ceph-mon[64115]: Updating smithi144:/etc/ceph/ceph.client.admin.keyring 2024-09-16T08:17:57.932 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:57 smithi028 ceph-mon[64115]: Updating smithi028:/etc/ceph/ceph.client.admin.keyring 2024-09-16T08:17:57.932 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:57 smithi028 ceph-mon[64115]: Updating smithi144:/var/lib/ceph/5622b2fe-7402-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-16T08:17:57.932 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:57 smithi028 ceph-mon[64115]: pgmap v8: 97 pgs: 97 active+clean; 326 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 24 KiB/s rd, 2.2 MiB/s wr, 227 op/s 2024-09-16T08:17:57.932 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:57 smithi028 ceph-mon[64115]: Updating smithi028:/var/lib/ceph/5622b2fe-7402-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-16T08:17:57.932 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:57 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:17:57.932 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:57 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:17:57.933 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:57 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:17:57.933 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:57 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:17:57.933 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:57 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:17:57.933 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:57 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-16T08:17:57.933 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:57 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "versions"} : dispatch 2024-09-16T08:17:57.933 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:57 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "quorum_status"} : dispatch 2024-09-16T08:17:58.035 INFO:teuthology.orchestra.run.smithi028.stdout: 5 137387 10.47 MB/sec execute 176 sec latency 44.106 ms 2024-09-16T08:17:58.082 INFO:teuthology.orchestra.run.smithi028.stdout:HEALTH_OK 2024-09-16T08:17:58.156 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:17:57 smithi144 ceph-mon[25668]: from='client.24655 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T08:17:58.156 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:17:57 smithi144 ceph-mon[25668]: Updating smithi028:/etc/ceph/ceph.conf 2024-09-16T08:17:58.156 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:17:57 smithi144 ceph-mon[25668]: Updating smithi144:/etc/ceph/ceph.conf 2024-09-16T08:17:58.156 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:17:57 smithi144 ceph-mon[25668]: Updating smithi144:/var/lib/ceph/5622b2fe-7402-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-16T08:17:58.157 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:17:57 smithi144 ceph-mon[25668]: Updating smithi028:/var/lib/ceph/5622b2fe-7402-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-16T08:17:58.157 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:17:57 smithi144 ceph-mon[25668]: Updating smithi144:/etc/ceph/ceph.client.admin.keyring 2024-09-16T08:17:58.157 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:17:57 smithi144 ceph-mon[25668]: Updating smithi028:/etc/ceph/ceph.client.admin.keyring 2024-09-16T08:17:58.157 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:17:57 smithi144 ceph-mon[25668]: Updating smithi144:/var/lib/ceph/5622b2fe-7402-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-16T08:17:58.157 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:17:57 smithi144 ceph-mon[25668]: pgmap v8: 97 pgs: 97 active+clean; 326 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 24 KiB/s rd, 2.2 MiB/s wr, 227 op/s 2024-09-16T08:17:58.157 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:17:57 smithi144 ceph-mon[25668]: Updating smithi028:/var/lib/ceph/5622b2fe-7402-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-16T08:17:58.157 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:17:57 smithi144 ceph-mon[25668]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:17:58.157 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:17:57 smithi144 ceph-mon[25668]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:17:58.157 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:17:57 smithi144 ceph-mon[25668]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:17:58.157 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:17:57 smithi144 ceph-mon[25668]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:17:58.157 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:17:57 smithi144 ceph-mon[25668]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:17:58.157 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:17:57 smithi144 ceph-mon[25668]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-16T08:17:58.157 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:17:57 smithi144 ceph-mon[25668]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "versions"} : dispatch 2024-09-16T08:17:58.158 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:17:57 smithi144 ceph-mon[25668]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "quorum_status"} : dispatch 2024-09-16T08:17:58.954 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:58 smithi028 ceph-mon[64115]: from='client.34148 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T08:17:58.954 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:58 smithi028 ceph-mon[64115]: from='client.? 172.21.15.28:0/3538562798' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-16T08:17:59.036 INFO:teuthology.orchestra.run.smithi028.stdout: 5 137827 10.48 MB/sec execute 177 sec latency 35.768 ms 2024-09-16T08:17:59.195 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:17:58 smithi144 ceph-mon[25668]: from='client.34148 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T08:17:59.196 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:17:58 smithi144 ceph-mon[25668]: from='client.? 172.21.15.28:0/3538562798' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-16T08:17:59.975 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:59 smithi028 ceph-mon[64115]: pgmap v9: 97 pgs: 97 active+clean; 329 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 22 KiB/s rd, 3.4 MiB/s wr, 372 op/s 2024-09-16T08:17:59.975 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:59 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:17:59.975 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:59 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-09-16T08:17:59.975 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:59 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-09-16T08:17:59.975 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:17:59 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-16T08:18:00.036 INFO:teuthology.orchestra.run.smithi028.stdout: 5 138105 10.48 MB/sec execute 178 sec latency 39.360 ms 2024-09-16T08:18:00.312 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:17:59 smithi144 ceph-mon[25668]: pgmap v9: 97 pgs: 97 active+clean; 329 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 22 KiB/s rd, 3.4 MiB/s wr, 372 op/s 2024-09-16T08:18:00.313 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:17:59 smithi144 ceph-mon[25668]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:18:00.313 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:17:59 smithi144 ceph-mon[25668]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-09-16T08:18:00.313 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:17:59 smithi144 ceph-mon[25668]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-09-16T08:18:00.313 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:17:59 smithi144 ceph-mon[25668]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-16T08:18:01.004 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:18:00 smithi028 ceph-mon[64115]: Upgrade: Updating mon.smithi144 2024-09-16T08:18:01.004 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:18:00 smithi028 ceph-mon[64115]: Deploying daemon mon.smithi144 on smithi144 2024-09-16T08:18:01.036 INFO:teuthology.orchestra.run.smithi028.stdout: 5 138405 10.48 MB/sec execute 179 sec latency 50.262 ms 2024-09-16T08:18:01.328 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:00 smithi144 ceph-mon[25668]: Upgrade: Updating mon.smithi144 2024-09-16T08:18:01.328 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:00 smithi144 ceph-mon[25668]: Deploying daemon mon.smithi144 on smithi144 2024-09-16T08:18:01.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:01 smithi144 systemd[1]: Stopping Ceph mon.smithi144 for 5622b2fe-7402-11ef-bceb-c7b262605968... 2024-09-16T08:18:02.036 INFO:teuthology.orchestra.run.smithi028.stdout: 5 138785 10.48 MB/sec execute 180 sec latency 61.817 ms 2024-09-16T08:18:02.081 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:01 smithi144 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-mon-smithi144[25650]: 2024-09-16T08:18:01.613+0000 7f41c9a5b700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.smithi144 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-09-16T08:18:02.081 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:01 smithi144 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-mon-smithi144[25650]: 2024-09-16T08:18:01.613+0000 7f41c9a5b700 -1 mon.smithi144@1(peon) e2 *** Got Signal Terminated *** 2024-09-16T08:18:02.081 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:01 smithi144 podman[48993]: 2024-09-16 08:18:01.836427067 +0000 UTC m=+0.353791143 container died 1c80e3ff3507e2252ad532518c038da66adb6f1edca0f539537e2e2f64828b22 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-5622b2fe-7402-11ef-bceb-c7b262605968-mon-smithi144, GIT_BRANCH=HEAD, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, distribution-scope=public, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, name=centos-stream, io.openshift.tags=base centos centos-stream, CEPH_POINT_RELEASE=-17.2.0, vcs-type=git, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, RELEASE=HEAD, vendor=Red Hat, Inc., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, architecture=x86_64, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_CLEAN=True, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.k8s.display-name=CentOS Stream 8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.buildah.version=1.19.8, version=8, maintainer=Guillaume Abrioux , build-date=2022-05-03T08:36:31.336870, io.openshift.expose-services=, ceph=True, com.redhat.component=centos-stream-container, release=754) 2024-09-16T08:18:02.721 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:02 smithi144 podman[48993]: 2024-09-16 08:18:02.469921059 +0000 UTC m=+0.987285130 container cleanup 1c80e3ff3507e2252ad532518c038da66adb6f1edca0f539537e2e2f64828b22 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-5622b2fe-7402-11ef-bceb-c7b262605968-mon-smithi144, CEPH_POINT_RELEASE=-17.2.0, GIT_BRANCH=HEAD, architecture=x86_64, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.openshift.tags=base centos centos-stream, build-date=2022-05-03T08:36:31.336870, io.openshift.expose-services=, name=centos-stream, com.redhat.component=centos-stream-container, ceph=True, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, release=754, RELEASE=HEAD, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.k8s.display-name=CentOS Stream 8, distribution-scope=public, io.buildah.version=1.19.8, vendor=Red Hat, Inc., GIT_CLEAN=True, GIT_REPO=https://github.com/ceph/ceph-container.git, version=8, maintainer=Guillaume Abrioux , vcs-type=git, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/) 2024-09-16T08:18:02.721 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:02 smithi144 bash[48993]: ceph-5622b2fe-7402-11ef-bceb-c7b262605968-mon-smithi144 2024-09-16T08:18:02.971 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:02 smithi144 podman[49006]: 2024-09-16 08:18:02.721696598 +0000 UTC m=+0.880673902 container remove 1c80e3ff3507e2252ad532518c038da66adb6f1edca0f539537e2e2f64828b22 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-5622b2fe-7402-11ef-bceb-c7b262605968-mon-smithi144, io.buildah.version=1.19.8, distribution-scope=public, io.k8s.display-name=CentOS Stream 8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_CLEAN=True, vendor=Red Hat, Inc., GIT_BRANCH=HEAD, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.component=centos-stream-container, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vcs-type=git, build-date=2022-05-03T08:36:31.336870, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., CEPH_POINT_RELEASE=-17.2.0, RELEASE=HEAD, maintainer=Guillaume Abrioux , architecture=x86_64, io.openshift.expose-services=, io.openshift.tags=base centos centos-stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_REPO=https://github.com/ceph/ceph-container.git, ceph=True, name=centos-stream, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, release=754, version=8) 2024-09-16T08:18:02.972 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:02 smithi144 systemd[1]: ceph-5622b2fe-7402-11ef-bceb-c7b262605968@mon.smithi144.service: Deactivated successfully. 2024-09-16T08:18:02.972 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:02 smithi144 systemd[1]: Stopped Ceph mon.smithi144 for 5622b2fe-7402-11ef-bceb-c7b262605968. 2024-09-16T08:18:02.972 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:02 smithi144 systemd[1]: ceph-5622b2fe-7402-11ef-bceb-c7b262605968@mon.smithi144.service: Consumed 6.975s CPU time. 2024-09-16T08:18:03.036 INFO:teuthology.orchestra.run.smithi028.stdout: 5 139138 10.47 MB/sec execute 181 sec latency 148.415 ms 2024-09-16T08:18:03.550 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:03 smithi144 systemd[1]: Starting Ceph mon.smithi144 for 5622b2fe-7402-11ef-bceb-c7b262605968... 2024-09-16T08:18:03.551 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:03 smithi144 podman[49130]: 2024-09-16 08:18:03.355607586 +0000 UTC m=+0.016344518 image pull 425e316ba8ebf74f04aa3640f7deb4fa3f293a78d0c8d097cb679eaf86ddca6c quay.ceph.io/ceph-ci/ceph@sha256:f7ad55f9ccd17ba66aadb19b4d291e67b60ea24e5d8a0c0e92941fd59c81e4ce 2024-09-16T08:18:03.551 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:03 smithi144 podman[49130]: 2024-09-16 08:18:03.551396698 +0000 UTC m=+0.212133619 container create c081c5ed44b540e028495bb1c8b3001db8efa379cb6417b527cbf19662a33ca7 (image=quay.ceph.io/ceph-ci/ceph@sha256:f7ad55f9ccd17ba66aadb19b4d291e67b60ea24e5d8a0c0e92941fd59c81e4ce, name=ceph-5622b2fe-7402-11ef-bceb-c7b262605968-mon-smithi144, io.buildah.version=1.37.2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, ceph=True, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, RELEASE=wip-guits-main-2024-09-13-1248-8293d73, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_BRANCH=HEAD) 2024-09-16T08:18:04.036 INFO:teuthology.orchestra.run.smithi028.stdout: 5 139463 10.44 MB/sec execute 182 sec latency 65.377 ms 2024-09-16T08:18:04.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:03 smithi144 podman[49130]: 2024-09-16 08:18:03.942772689 +0000 UTC m=+0.603509624 container init c081c5ed44b540e028495bb1c8b3001db8efa379cb6417b527cbf19662a33ca7 (image=quay.ceph.io/ceph-ci/ceph@sha256:f7ad55f9ccd17ba66aadb19b4d291e67b60ea24e5d8a0c0e92941fd59c81e4ce, name=ceph-5622b2fe-7402-11ef-bceb-c7b262605968-mon-smithi144, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, RELEASE=wip-guits-main-2024-09-13-1248-8293d73, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965) 2024-09-16T08:18:04.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:03 smithi144 podman[49130]: 2024-09-16 08:18:03.946037059 +0000 UTC m=+0.606773983 container start c081c5ed44b540e028495bb1c8b3001db8efa379cb6417b527cbf19662a33ca7 (image=quay.ceph.io/ceph-ci/ceph@sha256:f7ad55f9ccd17ba66aadb19b4d291e67b60ea24e5d8a0c0e92941fd59c81e4ce, name=ceph-5622b2fe-7402-11ef-bceb-c7b262605968-mon-smithi144, RELEASE=wip-guits-main-2024-09-13-1248-8293d73, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , ceph=True, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD) 2024-09-16T08:18:04.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:03 smithi144 ceph-mon[49149]: set uid:gid to 167:167 (ceph:ceph) 2024-09-16T08:18:04.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:03 smithi144 ceph-mon[49149]: ceph version 19.3.0-4917-g8293d73f (8293d73f8690540e843a81caec373f9cc29cf705) squid (dev), process ceph-mon, pid 2 2024-09-16T08:18:04.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:03 smithi144 ceph-mon[49149]: pidfile_write: ignore empty --pid-file 2024-09-16T08:18:04.333 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:03 smithi144 ceph-mon[49149]: load: jerasure load: lrc 2024-09-16T08:18:04.333 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:03 smithi144 ceph-mon[49149]: rocksdb: RocksDB version: 7.9.2 2024-09-16T08:18:04.333 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:03 smithi144 ceph-mon[49149]: rocksdb: Git sha 0 2024-09-16T08:18:04.333 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:03 smithi144 ceph-mon[49149]: rocksdb: Compile date 2024-09-13 14:50:41 2024-09-16T08:18:04.333 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:03 smithi144 ceph-mon[49149]: rocksdb: DB SUMMARY 2024-09-16T08:18:04.333 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:03 smithi144 ceph-mon[49149]: rocksdb: DB Session ID: Q1LDDHK5YZVGG7RZ5BPT 2024-09-16T08:18:04.333 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:03 smithi144 ceph-mon[49149]: rocksdb: CURRENT file: CURRENT 2024-09-16T08:18:04.333 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:03 smithi144 ceph-mon[49149]: rocksdb: IDENTITY file: IDENTITY 2024-09-16T08:18:04.333 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:03 smithi144 ceph-mon[49149]: rocksdb: MANIFEST file: MANIFEST-000009 size: 903 Bytes 2024-09-16T08:18:04.333 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:03 smithi144 ceph-mon[49149]: rocksdb: SST files in /var/lib/ceph/mon/ceph-smithi144/store.db dir, Total Num: 1, files: 000024.sst 2024-09-16T08:18:04.333 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:03 smithi144 ceph-mon[49149]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-smithi144/store.db: 000022.log size: 7204187 ; 2024-09-16T08:18:04.333 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:03 smithi144 ceph-mon[49149]: rocksdb: Options.error_if_exists: 0 2024-09-16T08:18:04.333 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:03 smithi144 ceph-mon[49149]: rocksdb: Options.create_if_missing: 0 2024-09-16T08:18:04.334 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:03 smithi144 ceph-mon[49149]: rocksdb: Options.paranoid_checks: 1 2024-09-16T08:18:04.334 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:03 smithi144 ceph-mon[49149]: rocksdb: Options.flush_verify_memtable_count: 1 2024-09-16T08:18:04.334 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:03 smithi144 ceph-mon[49149]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-09-16T08:18:04.334 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:03 smithi144 ceph-mon[49149]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-09-16T08:18:04.334 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:03 smithi144 ceph-mon[49149]: rocksdb: Options.env: 0x561e29406120 2024-09-16T08:18:04.334 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:03 smithi144 ceph-mon[49149]: rocksdb: Options.fs: PosixFileSystem 2024-09-16T08:18:04.334 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:03 smithi144 ceph-mon[49149]: rocksdb: Options.info_log: 0x561e2b9f98e0 2024-09-16T08:18:04.334 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:03 smithi144 ceph-mon[49149]: rocksdb: Options.max_file_opening_threads: 16 2024-09-16T08:18:04.335 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:03 smithi144 ceph-mon[49149]: rocksdb: Options.statistics: (nil) 2024-09-16T08:18:04.335 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:03 smithi144 ceph-mon[49149]: rocksdb: Options.use_fsync: 0 2024-09-16T08:18:04.335 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:03 smithi144 ceph-mon[49149]: rocksdb: Options.max_log_file_size: 0 2024-09-16T08:18:04.335 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:03 smithi144 ceph-mon[49149]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-09-16T08:18:04.335 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:03 smithi144 ceph-mon[49149]: rocksdb: Options.log_file_time_to_roll: 0 2024-09-16T08:18:04.336 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:03 smithi144 ceph-mon[49149]: rocksdb: Options.keep_log_file_num: 1000 2024-09-16T08:18:04.336 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:03 smithi144 ceph-mon[49149]: rocksdb: Options.recycle_log_file_num: 0 2024-09-16T08:18:04.336 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:03 smithi144 ceph-mon[49149]: rocksdb: Options.allow_fallocate: 1 2024-09-16T08:18:04.336 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:03 smithi144 ceph-mon[49149]: rocksdb: Options.allow_mmap_reads: 0 2024-09-16T08:18:04.336 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:03 smithi144 ceph-mon[49149]: rocksdb: Options.allow_mmap_writes: 0 2024-09-16T08:18:04.336 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:03 smithi144 ceph-mon[49149]: rocksdb: Options.use_direct_reads: 0 2024-09-16T08:18:04.336 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:03 smithi144 ceph-mon[49149]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-09-16T08:18:04.336 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:03 smithi144 ceph-mon[49149]: rocksdb: Options.create_missing_column_families: 0 2024-09-16T08:18:04.336 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:03 smithi144 ceph-mon[49149]: rocksdb: Options.db_log_dir: 2024-09-16T08:18:04.336 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:03 smithi144 ceph-mon[49149]: rocksdb: Options.wal_dir: 2024-09-16T08:18:04.336 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:03 smithi144 ceph-mon[49149]: rocksdb: Options.table_cache_numshardbits: 6 2024-09-16T08:18:04.336 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:03 smithi144 ceph-mon[49149]: rocksdb: Options.WAL_ttl_seconds: 0 2024-09-16T08:18:04.337 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:03 smithi144 ceph-mon[49149]: rocksdb: Options.WAL_size_limit_MB: 0 2024-09-16T08:18:04.337 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:03 smithi144 ceph-mon[49149]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-09-16T08:18:04.337 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:03 smithi144 ceph-mon[49149]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-09-16T08:18:04.337 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:03 smithi144 ceph-mon[49149]: rocksdb: Options.is_fd_close_on_exec: 1 2024-09-16T08:18:04.337 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:03 smithi144 ceph-mon[49149]: rocksdb: Options.advise_random_on_open: 1 2024-09-16T08:18:04.337 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:03 smithi144 ceph-mon[49149]: rocksdb: Options.db_write_buffer_size: 0 2024-09-16T08:18:04.337 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:03 smithi144 ceph-mon[49149]: rocksdb: Options.write_buffer_manager: 0x561e2b9fdb80 2024-09-16T08:18:04.337 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:03 smithi144 ceph-mon[49149]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-09-16T08:18:04.337 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:03 smithi144 ceph-mon[49149]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-09-16T08:18:04.337 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:03 smithi144 ceph-mon[49149]: rocksdb: Options.use_adaptive_mutex: 0 2024-09-16T08:18:04.337 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:03 smithi144 ceph-mon[49149]: rocksdb: Options.rate_limiter: (nil) 2024-09-16T08:18:04.337 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:03 smithi144 ceph-mon[49149]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-09-16T08:18:04.337 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:03 smithi144 ceph-mon[49149]: rocksdb: Options.wal_recovery_mode: 2 2024-09-16T08:18:04.338 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:03 smithi144 ceph-mon[49149]: rocksdb: Options.enable_thread_tracking: 0 2024-09-16T08:18:04.338 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:03 smithi144 ceph-mon[49149]: rocksdb: Options.enable_pipelined_write: 0 2024-09-16T08:18:04.338 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:03 smithi144 ceph-mon[49149]: rocksdb: Options.unordered_write: 0 2024-09-16T08:18:04.338 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:03 smithi144 ceph-mon[49149]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-09-16T08:18:04.338 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:03 smithi144 ceph-mon[49149]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-09-16T08:18:04.338 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:03 smithi144 ceph-mon[49149]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-09-16T08:18:04.338 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:03 smithi144 ceph-mon[49149]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-09-16T08:18:04.338 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:03 smithi144 ceph-mon[49149]: rocksdb: Options.row_cache: None 2024-09-16T08:18:04.338 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:03 smithi144 ceph-mon[49149]: rocksdb: Options.wal_filter: None 2024-09-16T08:18:04.338 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:03 smithi144 ceph-mon[49149]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-09-16T08:18:04.338 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:03 smithi144 ceph-mon[49149]: rocksdb: Options.allow_ingest_behind: 0 2024-09-16T08:18:04.338 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:03 smithi144 ceph-mon[49149]: rocksdb: Options.two_write_queues: 0 2024-09-16T08:18:04.338 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:03 smithi144 ceph-mon[49149]: rocksdb: Options.manual_wal_flush: 0 2024-09-16T08:18:04.339 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:03 smithi144 ceph-mon[49149]: rocksdb: Options.wal_compression: 0 2024-09-16T08:18:04.339 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:03 smithi144 ceph-mon[49149]: rocksdb: Options.atomic_flush: 0 2024-09-16T08:18:04.339 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:03 smithi144 ceph-mon[49149]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-09-16T08:18:04.339 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:03 smithi144 ceph-mon[49149]: rocksdb: Options.persist_stats_to_disk: 0 2024-09-16T08:18:04.339 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:03 smithi144 ceph-mon[49149]: rocksdb: Options.write_dbid_to_manifest: 0 2024-09-16T08:18:04.339 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:03 smithi144 ceph-mon[49149]: rocksdb: Options.log_readahead_size: 0 2024-09-16T08:18:04.339 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:03 smithi144 ceph-mon[49149]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-09-16T08:18:04.339 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:03 smithi144 ceph-mon[49149]: rocksdb: Options.best_efforts_recovery: 0 2024-09-16T08:18:04.339 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:03 smithi144 ceph-mon[49149]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-09-16T08:18:04.339 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:03 smithi144 ceph-mon[49149]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-09-16T08:18:04.339 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:03 smithi144 ceph-mon[49149]: rocksdb: Options.allow_data_in_errors: 0 2024-09-16T08:18:04.339 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:03 smithi144 ceph-mon[49149]: rocksdb: Options.db_host_id: __hostname__ 2024-09-16T08:18:04.339 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:03 smithi144 ceph-mon[49149]: rocksdb: Options.enforce_single_del_contracts: true 2024-09-16T08:18:04.339 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:03 smithi144 ceph-mon[49149]: rocksdb: Options.max_background_jobs: 2 2024-09-16T08:18:04.339 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:03 smithi144 ceph-mon[49149]: rocksdb: Options.max_background_compactions: -1 2024-09-16T08:18:04.340 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:03 smithi144 ceph-mon[49149]: rocksdb: Options.max_subcompactions: 1 2024-09-16T08:18:04.340 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:03 smithi144 ceph-mon[49149]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-09-16T08:18:04.340 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:03 smithi144 ceph-mon[49149]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-09-16T08:18:04.340 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:03 smithi144 ceph-mon[49149]: rocksdb: Options.delayed_write_rate : 16777216 2024-09-16T08:18:04.340 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:03 smithi144 ceph-mon[49149]: rocksdb: Options.max_total_wal_size: 0 2024-09-16T08:18:04.340 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:03 smithi144 ceph-mon[49149]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-09-16T08:18:04.340 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:03 smithi144 ceph-mon[49149]: rocksdb: Options.stats_dump_period_sec: 600 2024-09-16T08:18:04.340 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:03 smithi144 ceph-mon[49149]: rocksdb: Options.stats_persist_period_sec: 600 2024-09-16T08:18:04.340 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:03 smithi144 ceph-mon[49149]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-09-16T08:18:04.340 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:03 smithi144 ceph-mon[49149]: rocksdb: Options.max_open_files: -1 2024-09-16T08:18:04.340 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:03 smithi144 ceph-mon[49149]: rocksdb: Options.bytes_per_sync: 0 2024-09-16T08:18:04.340 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:03 smithi144 ceph-mon[49149]: rocksdb: Options.wal_bytes_per_sync: 0 2024-09-16T08:18:04.340 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:03 smithi144 ceph-mon[49149]: rocksdb: Options.strict_bytes_per_sync: 0 2024-09-16T08:18:04.340 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:03 smithi144 ceph-mon[49149]: rocksdb: Options.compaction_readahead_size: 0 2024-09-16T08:18:04.340 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:03 smithi144 ceph-mon[49149]: rocksdb: Options.max_background_flushes: -1 2024-09-16T08:18:04.341 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:03 smithi144 ceph-mon[49149]: rocksdb: Compression algorithms supported: 2024-09-16T08:18:04.341 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:03 smithi144 ceph-mon[49149]: rocksdb: kZSTD supported: 0 2024-09-16T08:18:04.341 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:03 smithi144 ceph-mon[49149]: rocksdb: kXpressCompression supported: 0 2024-09-16T08:18:04.341 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:03 smithi144 ceph-mon[49149]: rocksdb: kBZip2Compression supported: 0 2024-09-16T08:18:04.341 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:03 smithi144 ceph-mon[49149]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-09-16T08:18:04.341 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:03 smithi144 ceph-mon[49149]: rocksdb: kLZ4Compression supported: 1 2024-09-16T08:18:04.341 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:03 smithi144 ceph-mon[49149]: rocksdb: kZlibCompression supported: 1 2024-09-16T08:18:04.341 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:03 smithi144 ceph-mon[49149]: rocksdb: kLZ4HCCompression supported: 1 2024-09-16T08:18:04.341 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:03 smithi144 ceph-mon[49149]: rocksdb: kSnappyCompression supported: 1 2024-09-16T08:18:04.342 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:03 smithi144 ceph-mon[49149]: rocksdb: Fast CRC32 supported: Supported on x86 2024-09-16T08:18:04.342 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:03 smithi144 ceph-mon[49149]: rocksdb: DMutex implementation: pthread_mutex_t 2024-09-16T08:18:04.342 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:03 smithi144 ceph-mon[49149]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-smithi144/store.db/MANIFEST-000009 2024-09-16T08:18:04.342 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:03 smithi144 ceph-mon[49149]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-09-16T08:18:04.342 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:03 smithi144 ceph-mon[49149]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-09-16T08:18:04.342 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:03 smithi144 ceph-mon[49149]: rocksdb: Options.merge_operator: 2024-09-16T08:18:04.342 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:03 smithi144 ceph-mon[49149]: rocksdb: Options.compaction_filter: None 2024-09-16T08:18:04.342 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:03 smithi144 ceph-mon[49149]: rocksdb: Options.compaction_filter_factory: None 2024-09-16T08:18:04.342 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:03 smithi144 ceph-mon[49149]: rocksdb: Options.sst_partitioner_factory: None 2024-09-16T08:18:04.342 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:03 smithi144 ceph-mon[49149]: rocksdb: Options.memtable_factory: SkipListFactory 2024-09-16T08:18:04.342 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:03 smithi144 ceph-mon[49149]: rocksdb: Options.table_factory: BlockBasedTable 2024-09-16T08:18:04.342 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:03 smithi144 ceph-mon[49149]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x561e2b9f9540) 2024-09-16T08:18:04.342 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: cache_index_and_filter_blocks: 1 2024-09-16T08:18:04.343 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-09-16T08:18:04.343 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-09-16T08:18:04.343 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: pin_top_level_index_and_filter: 1 2024-09-16T08:18:04.343 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: index_type: 0 2024-09-16T08:18:04.343 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: data_block_index_type: 0 2024-09-16T08:18:04.343 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: index_shortening: 1 2024-09-16T08:18:04.343 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: data_block_hash_table_util_ratio: 0.750000 2024-09-16T08:18:04.343 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: checksum: 4 2024-09-16T08:18:04.343 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: no_block_cache: 0 2024-09-16T08:18:04.343 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: block_cache: 0x561e2b9ed610 2024-09-16T08:18:04.343 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: block_cache_name: BinnedLRUCache 2024-09-16T08:18:04.343 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: block_cache_options: 2024-09-16T08:18:04.343 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: capacity : 536870912 2024-09-16T08:18:04.343 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: num_shard_bits : 4 2024-09-16T08:18:04.344 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: strict_capacity_limit : 0 2024-09-16T08:18:04.344 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: high_pri_pool_ratio: 0.000 2024-09-16T08:18:04.344 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: block_cache_compressed: (nil) 2024-09-16T08:18:04.344 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: persistent_cache: (nil) 2024-09-16T08:18:04.344 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: block_size: 4096 2024-09-16T08:18:04.344 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: block_size_deviation: 10 2024-09-16T08:18:04.344 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: block_restart_interval: 16 2024-09-16T08:18:04.344 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: index_block_restart_interval: 1 2024-09-16T08:18:04.344 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: metadata_block_size: 4096 2024-09-16T08:18:04.344 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: partition_filters: 0 2024-09-16T08:18:04.344 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: use_delta_encoding: 1 2024-09-16T08:18:04.344 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: filter_policy: bloomfilter 2024-09-16T08:18:04.344 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: whole_key_filtering: 1 2024-09-16T08:18:04.344 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: verify_compression: 0 2024-09-16T08:18:04.344 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: read_amp_bytes_per_bit: 0 2024-09-16T08:18:04.344 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: format_version: 5 2024-09-16T08:18:04.345 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: enable_index_compression: 1 2024-09-16T08:18:04.345 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: block_align: 0 2024-09-16T08:18:04.345 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: max_auto_readahead_size: 262144 2024-09-16T08:18:04.345 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: prepopulate_block_cache: 0 2024-09-16T08:18:04.345 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: initial_auto_readahead_size: 8192 2024-09-16T08:18:04.345 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: num_file_reads_for_auto_readahead: 2 2024-09-16T08:18:04.345 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:03 smithi144 ceph-mon[49149]: rocksdb: Options.write_buffer_size: 33554432 2024-09-16T08:18:04.345 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:03 smithi144 ceph-mon[49149]: rocksdb: Options.max_write_buffer_number: 2 2024-09-16T08:18:04.345 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:03 smithi144 ceph-mon[49149]: rocksdb: Options.compression: NoCompression 2024-09-16T08:18:04.345 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:03 smithi144 ceph-mon[49149]: rocksdb: Options.bottommost_compression: Disabled 2024-09-16T08:18:04.345 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:03 smithi144 ceph-mon[49149]: rocksdb: Options.prefix_extractor: nullptr 2024-09-16T08:18:04.345 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:03 smithi144 ceph-mon[49149]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-09-16T08:18:04.345 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:03 smithi144 ceph-mon[49149]: rocksdb: Options.num_levels: 7 2024-09-16T08:18:04.345 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:03 smithi144 ceph-mon[49149]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-09-16T08:18:04.345 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:03 smithi144 ceph-mon[49149]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-09-16T08:18:04.346 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:03 smithi144 ceph-mon[49149]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-09-16T08:18:04.346 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:03 smithi144 ceph-mon[49149]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-09-16T08:18:04.346 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:03 smithi144 ceph-mon[49149]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-09-16T08:18:04.346 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:03 smithi144 ceph-mon[49149]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-09-16T08:18:04.346 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:03 smithi144 ceph-mon[49149]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-09-16T08:18:04.346 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:03 smithi144 ceph-mon[49149]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-09-16T08:18:04.346 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:03 smithi144 ceph-mon[49149]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-09-16T08:18:04.346 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:03 smithi144 ceph-mon[49149]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-09-16T08:18:04.346 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:03 smithi144 ceph-mon[49149]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-09-16T08:18:04.346 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:03 smithi144 ceph-mon[49149]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-09-16T08:18:04.346 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:03 smithi144 ceph-mon[49149]: rocksdb: Options.compression_opts.window_bits: -14 2024-09-16T08:18:04.346 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:03 smithi144 ceph-mon[49149]: rocksdb: Options.compression_opts.level: 32767 2024-09-16T08:18:04.346 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:03 smithi144 ceph-mon[49149]: rocksdb: Options.compression_opts.strategy: 0 2024-09-16T08:18:04.346 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:03 smithi144 ceph-mon[49149]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-09-16T08:18:04.346 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:03 smithi144 ceph-mon[49149]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-09-16T08:18:04.347 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:03 smithi144 ceph-mon[49149]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-09-16T08:18:04.347 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:03 smithi144 ceph-mon[49149]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-09-16T08:18:04.347 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:03 smithi144 ceph-mon[49149]: rocksdb: Options.compression_opts.enabled: false 2024-09-16T08:18:04.347 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:03 smithi144 ceph-mon[49149]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-09-16T08:18:04.347 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:03 smithi144 ceph-mon[49149]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-09-16T08:18:04.347 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:03 smithi144 ceph-mon[49149]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-09-16T08:18:04.347 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:03 smithi144 ceph-mon[49149]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-09-16T08:18:04.347 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:03 smithi144 ceph-mon[49149]: rocksdb: Options.target_file_size_base: 67108864 2024-09-16T08:18:04.347 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:03 smithi144 ceph-mon[49149]: rocksdb: Options.target_file_size_multiplier: 1 2024-09-16T08:18:04.347 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:03 smithi144 ceph-mon[49149]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-09-16T08:18:04.347 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:03 smithi144 ceph-mon[49149]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-09-16T08:18:04.347 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:03 smithi144 ceph-mon[49149]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-09-16T08:18:04.347 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:03 smithi144 ceph-mon[49149]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-09-16T08:18:04.347 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:03 smithi144 ceph-mon[49149]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-09-16T08:18:04.347 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:03 smithi144 ceph-mon[49149]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-09-16T08:18:04.348 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:03 smithi144 ceph-mon[49149]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-09-16T08:18:04.348 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:03 smithi144 ceph-mon[49149]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-09-16T08:18:04.348 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:03 smithi144 ceph-mon[49149]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-09-16T08:18:04.348 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:03 smithi144 ceph-mon[49149]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-09-16T08:18:04.348 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:03 smithi144 ceph-mon[49149]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-09-16T08:18:04.348 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:03 smithi144 ceph-mon[49149]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-09-16T08:18:04.348 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:03 smithi144 ceph-mon[49149]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-09-16T08:18:04.348 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:03 smithi144 ceph-mon[49149]: rocksdb: Options.arena_block_size: 1048576 2024-09-16T08:18:04.348 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:03 smithi144 ceph-mon[49149]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-09-16T08:18:04.348 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:03 smithi144 ceph-mon[49149]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-09-16T08:18:04.348 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:03 smithi144 ceph-mon[49149]: rocksdb: Options.disable_auto_compactions: 0 2024-09-16T08:18:04.348 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:03 smithi144 ceph-mon[49149]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-09-16T08:18:04.348 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:03 smithi144 ceph-mon[49149]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-09-16T08:18:04.348 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:03 smithi144 ceph-mon[49149]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-09-16T08:18:04.349 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:03 smithi144 ceph-mon[49149]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-09-16T08:18:04.349 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:03 smithi144 ceph-mon[49149]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-09-16T08:18:04.349 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:03 smithi144 ceph-mon[49149]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-09-16T08:18:04.349 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:03 smithi144 ceph-mon[49149]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-09-16T08:18:04.349 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:03 smithi144 ceph-mon[49149]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-09-16T08:18:04.349 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:03 smithi144 ceph-mon[49149]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-09-16T08:18:04.349 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:03 smithi144 ceph-mon[49149]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-09-16T08:18:04.349 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:03 smithi144 ceph-mon[49149]: rocksdb: Options.table_properties_collectors: CompactOnDeletionCollector (Sliding window size = 32768 Deletion trigger = 16384 Deletion ratio = 0); 2024-09-16T08:18:04.349 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:03 smithi144 ceph-mon[49149]: rocksdb: Options.inplace_update_support: 0 2024-09-16T08:18:04.349 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:03 smithi144 ceph-mon[49149]: rocksdb: Options.inplace_update_num_locks: 10000 2024-09-16T08:18:04.349 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:03 smithi144 ceph-mon[49149]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-09-16T08:18:04.349 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:03 smithi144 ceph-mon[49149]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-09-16T08:18:04.349 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:03 smithi144 ceph-mon[49149]: rocksdb: Options.memtable_huge_page_size: 0 2024-09-16T08:18:04.350 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:03 smithi144 ceph-mon[49149]: rocksdb: Options.bloom_locality: 0 2024-09-16T08:18:04.350 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:03 smithi144 ceph-mon[49149]: rocksdb: Options.max_successive_merges: 0 2024-09-16T08:18:04.350 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:03 smithi144 ceph-mon[49149]: rocksdb: Options.optimize_filters_for_hits: 0 2024-09-16T08:18:04.350 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:03 smithi144 ceph-mon[49149]: rocksdb: Options.paranoid_file_checks: 0 2024-09-16T08:18:04.350 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:03 smithi144 ceph-mon[49149]: rocksdb: Options.force_consistency_checks: 1 2024-09-16T08:18:04.350 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:03 smithi144 ceph-mon[49149]: rocksdb: Options.report_bg_io_stats: 0 2024-09-16T08:18:04.350 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:03 smithi144 ceph-mon[49149]: rocksdb: Options.ttl: 2592000 2024-09-16T08:18:04.350 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:03 smithi144 ceph-mon[49149]: rocksdb: Options.periodic_compaction_seconds: 0 2024-09-16T08:18:04.350 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:03 smithi144 ceph-mon[49149]: rocksdb: Options.preclude_last_level_data_seconds: 0 2024-09-16T08:18:04.351 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:03 smithi144 ceph-mon[49149]: rocksdb: Options.preserve_internal_time_seconds: 0 2024-09-16T08:18:04.351 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:03 smithi144 ceph-mon[49149]: rocksdb: Options.enable_blob_files: false 2024-09-16T08:18:04.351 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:03 smithi144 ceph-mon[49149]: rocksdb: Options.min_blob_size: 0 2024-09-16T08:18:04.351 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:03 smithi144 ceph-mon[49149]: rocksdb: Options.blob_file_size: 268435456 2024-09-16T08:18:04.351 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:03 smithi144 ceph-mon[49149]: rocksdb: Options.blob_compression_type: NoCompression 2024-09-16T08:18:04.351 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:03 smithi144 ceph-mon[49149]: rocksdb: Options.enable_blob_garbage_collection: false 2024-09-16T08:18:04.351 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:03 smithi144 ceph-mon[49149]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-09-16T08:18:04.351 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:03 smithi144 ceph-mon[49149]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-09-16T08:18:04.351 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:03 smithi144 ceph-mon[49149]: rocksdb: Options.blob_compaction_readahead_size: 0 2024-09-16T08:18:04.352 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:03 smithi144 ceph-mon[49149]: rocksdb: Options.blob_file_starting_level: 0 2024-09-16T08:18:04.352 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:03 smithi144 ceph-mon[49149]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-09-16T08:18:04.352 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:03 smithi144 ceph-mon[49149]: rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 24.sst 2024-09-16T08:18:04.352 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:03 smithi144 ceph-mon[49149]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-09-16T08:18:04.352 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:03 smithi144 ceph-mon[49149]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-smithi144/store.db/MANIFEST-000009 succeeded,manifest_file_number is 9, next_file_number is 26, last_sequence is 12281, log_number is 22,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-09-16T08:18:04.352 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:03 smithi144 ceph-mon[49149]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 22 2024-09-16T08:18:04.352 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:03 smithi144 ceph-mon[49149]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: 87023ea9-601f-4750-8b2f-aaafc54831fc 2024-09-16T08:18:04.352 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:03 smithi144 ceph-mon[49149]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726474683974062, "job": 1, "event": "recovery_started", "wal_files": [22]} 2024-09-16T08:18:04.352 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:03 smithi144 ceph-mon[49149]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #22 mode 2 2024-09-16T08:18:04.352 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:03 smithi144 ceph-mon[49149]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726474683991072, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 27, "file_size": 4408093, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 12286, "largest_seqno": 13834, "table_properties": {"data_size": 4400966, "index_size": 4482, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 1605, "raw_key_size": 14667, "raw_average_key_size": 23, "raw_value_size": 4387248, "raw_average_value_size": 6974, "num_data_blocks": 203, "num_entries": 629, "num_filter_entries": 629, "num_deletions": 2, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[CompactOnDeletionCollector]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1726474683, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "87023ea9-601f-4750-8b2f-aaafc54831fc", "db_session_id": "Q1LDDHK5YZVGG7RZ5BPT", "orig_file_number": 27, "seqno_to_time_mapping": "N/A"}} 2024-09-16T08:18:04.352 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:03 smithi144 ceph-mon[49149]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726474683991239, "job": 1, "event": "recovery_finished"} 2024-09-16T08:18:04.352 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:03 smithi144 ceph-mon[49149]: rocksdb: [db/version_set.cc:5047] Creating manifest 29 2024-09-16T08:18:04.353 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:03 smithi144 ceph-mon[49149]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-09-16T08:18:04.353 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:03 smithi144 ceph-mon[49149]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-smithi144/store.db/000022.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-09-16T08:18:04.353 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:03 smithi144 ceph-mon[49149]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x561e2ba1ce00 2024-09-16T08:18:04.353 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:03 smithi144 ceph-mon[49149]: rocksdb: DB pointer 0x561e2bb1e000 2024-09-16T08:18:04.353 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:03 smithi144 ceph-mon[49149]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-09-16T08:18:04.353 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:03 smithi144 ceph-mon[49149]: rocksdb: [db/db_impl/db_impl.cc:1111] 2024-09-16T08:18:04.353 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: ** DB Stats ** 2024-09-16T08:18:04.353 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-16T08:18:04.353 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-09-16T08:18:04.353 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-16T08:18:04.353 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-16T08:18:04.353 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-09-16T08:18:04.353 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-16T08:18:04.354 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-16T08:18:04.354 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: 2024-09-16T08:18:04.354 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: ** Compaction Stats [default] ** 2024-09-16T08:18:04.354 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-09-16T08:18:04.354 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-09-16T08:18:04.354 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: L0 1/0 4.20 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 532.5 0.01 0.00 1 0.008 0 0 0.0 0.0 2024-09-16T08:18:04.354 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: L6 1/0 7.02 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2024-09-16T08:18:04.354 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: Sum 2/0 11.23 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 532.5 0.01 0.00 1 0.008 0 0 0.0 0.0 2024-09-16T08:18:04.354 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 532.5 0.01 0.00 1 0.008 0 0 0.0 0.0 2024-09-16T08:18:04.354 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: 2024-09-16T08:18:04.355 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: ** Compaction Stats [default] ** 2024-09-16T08:18:04.355 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-09-16T08:18:04.355 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-16T08:18:04.355 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 532.5 0.01 0.00 1 0.008 0 0 0.0 0.0 2024-09-16T08:18:04.355 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: 2024-09-16T08:18:04.355 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-09-16T08:18:04.355 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: 2024-09-16T08:18:04.355 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-16T08:18:04.355 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: Flush(GB): cumulative 0.004, interval 0.004 2024-09-16T08:18:04.355 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-09-16T08:18:04.355 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-09-16T08:18:04.355 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-09-16T08:18:04.356 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: AddFile(Keys): cumulative 0, interval 0 2024-09-16T08:18:04.356 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: Cumulative compaction: 0.00 GB write, 192.52 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-16T08:18:04.356 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: Interval compaction: 0.00 GB write, 192.52 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-16T08:18:04.356 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-09-16T08:18:04.356 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: Block cache BinnedLRUCache@0x561e2b9ed610#2 capacity: 512.00 MB usage: 6.36 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1e-05 secs_since: 0 2024-09-16T08:18:04.356 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,1.78 KB,0.000339746%) IndexBlock(1,4.58 KB,0.000873208%) Misc(1,0.00 KB,0%) 2024-09-16T08:18:04.356 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: 2024-09-16T08:18:04.356 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: ** File Read Latency Histogram By Level [default] ** 2024-09-16T08:18:04.356 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:03 smithi144 ceph-mon[49149]: starting mon.smithi144 rank 1 at public addrs [v2:172.21.15.144:3300/0,v1:172.21.15.144:6789/0] at bind addrs [v2:172.21.15.144:3300/0,v1:172.21.15.144:6789/0] mon_data /var/lib/ceph/mon/ceph-smithi144 fsid 5622b2fe-7402-11ef-bceb-c7b262605968 2024-09-16T08:18:04.356 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:03 smithi144 ceph-mon[49149]: mon.smithi144@-1(???) e2 preinit fsid 5622b2fe-7402-11ef-bceb-c7b262605968 2024-09-16T08:18:04.356 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:03 smithi144 ceph-mon[49149]: mon.smithi144@-1(???).mds e7 new map 2024-09-16T08:18:04.356 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:03 smithi144 ceph-mon[49149]: mon.smithi144@-1(???).mds e7 print_map 2024-09-16T08:18:04.356 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: e7 2024-09-16T08:18:04.357 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: btime 1970-01-01T00:00:00:000000+0000 2024-09-16T08:18:04.357 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-09-16T08:18:04.357 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-09-16T08:18:04.357 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: legacy client fscid: 1 2024-09-16T08:18:04.357 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: 2024-09-16T08:18:04.357 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: Filesystem 'foofs' (1) 2024-09-16T08:18:04.357 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: fs_name foofs 2024-09-16T08:18:04.357 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: epoch 5 2024-09-16T08:18:04.357 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: flags 12 joinable allow_snaps allow_multimds_snaps 2024-09-16T08:18:04.357 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: created 2024-09-16T08:12:22.241379+0000 2024-09-16T08:18:04.357 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: modified 2024-09-16T08:12:28.262657+0000 2024-09-16T08:18:04.357 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: tableserver 0 2024-09-16T08:18:04.357 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: root 0 2024-09-16T08:18:04.358 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: session_timeout 60 2024-09-16T08:18:04.358 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: session_autoclose 300 2024-09-16T08:18:04.358 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: max_file_size 1099511627776 2024-09-16T08:18:04.358 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: max_xattr_size 65536 2024-09-16T08:18:04.358 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: required_client_features {} 2024-09-16T08:18:04.358 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: last_failure 0 2024-09-16T08:18:04.358 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: last_failure_osd_epoch 0 2024-09-16T08:18:04.358 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-09-16T08:18:04.358 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: max_mds 1 2024-09-16T08:18:04.358 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: in 0 2024-09-16T08:18:04.358 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: up {0=14480} 2024-09-16T08:18:04.358 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: failed 2024-09-16T08:18:04.358 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: damaged 2024-09-16T08:18:04.359 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: stopped 2024-09-16T08:18:04.359 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: data_pools [3] 2024-09-16T08:18:04.359 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: metadata_pool 2 2024-09-16T08:18:04.359 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: inline_data disabled 2024-09-16T08:18:04.359 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: balancer 2024-09-16T08:18:04.359 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: bal_rank_mask -1 2024-09-16T08:18:04.359 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: standby_count_wanted 1 2024-09-16T08:18:04.359 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: qdb_cluster leader: 0 members: 2024-09-16T08:18:04.359 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: [mds.foofs.smithi028.srtgev{0:14480} state up:active seq 2 join_fscid=1 addr [v2:172.21.15.28:6834/800761142,v1:172.21.15.28:6835/800761142] compat {c=[1],r=[1],i=[7ff]}] 2024-09-16T08:18:04.359 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: 2024-09-16T08:18:04.359 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: 2024-09-16T08:18:04.359 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: Standby daemons: 2024-09-16T08:18:04.359 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: 2024-09-16T08:18:04.360 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: [mds.foofs.smithi144.oozgsi{-1:24297} state up:standby seq 1 join_fscid=1 addr [v2:172.21.15.144:6832/2267142930,v1:172.21.15.144:6833/2267142930] compat {c=[1],r=[1],i=[7ff]}] 2024-09-16T08:18:04.360 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:03 smithi144 ceph-mon[49149]: mon.smithi144@-1(???).osd e72 crush map has features 3314933000854323200, adjusting msgr requires 2024-09-16T08:18:04.360 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:03 smithi144 ceph-mon[49149]: mon.smithi144@-1(???).osd e72 crush map has features 432629239337189376, adjusting msgr requires 2024-09-16T08:18:04.360 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:03 smithi144 ceph-mon[49149]: mon.smithi144@-1(???).osd e72 crush map has features 432629239337189376, adjusting msgr requires 2024-09-16T08:18:04.360 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:03 smithi144 ceph-mon[49149]: mon.smithi144@-1(???).osd e72 crush map has features 432629239337189376, adjusting msgr requires 2024-09-16T08:18:04.360 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:03 smithi144 ceph-mon[49149]: mon.smithi144@-1(???).paxosservice(auth 1..30) refresh upgraded, format 0 -> 3 2024-09-16T08:18:04.360 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:04 smithi144 bash[49130]: c081c5ed44b540e028495bb1c8b3001db8efa379cb6417b527cbf19662a33ca7 2024-09-16T08:18:04.360 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:04 smithi144 systemd[1]: Started Ceph mon.smithi144 for 5622b2fe-7402-11ef-bceb-c7b262605968. 2024-09-16T08:18:05.036 INFO:teuthology.orchestra.run.smithi028.stdout: 5 139859 10.43 MB/sec execute 183 sec latency 60.887 ms 2024-09-16T08:18:05.069 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:18:04 smithi028 ceph-mon[64115]: pgmap v10: 97 pgs: 97 active+clean; 329 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 20 KiB/s rd, 3.1 MiB/s wr, 339 op/s 2024-09-16T08:18:05.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:04 smithi144 ceph-mon[49149]: pgmap v10: 97 pgs: 97 active+clean; 329 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 20 KiB/s rd, 3.1 MiB/s wr, 339 op/s 2024-09-16T08:18:06.036 INFO:teuthology.orchestra.run.smithi028.stdout: 5 140326 10.43 MB/sec execute 184 sec latency 60.307 ms 2024-09-16T08:18:06.088 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:18:05 smithi028 ceph-mon[64115]: pgmap v12: 97 pgs: 97 active+clean; 338 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 23 KiB/s rd, 5.5 MiB/s wr, 550 op/s 2024-09-16T08:18:06.088 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:18:05 smithi028 ceph-mon[64115]: overall HEALTH_OK 2024-09-16T08:18:06.088 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:18:05 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "mon metadata", "id": "smithi028"} : dispatch 2024-09-16T08:18:06.088 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:18:05 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "mon metadata", "id": "smithi144"} : dispatch 2024-09-16T08:18:06.089 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:18:05 smithi028 ceph-mon[64115]: mon.smithi028 calling monitor election 2024-09-16T08:18:06.089 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:18:05 smithi028 ceph-mon[64115]: mon.smithi144 calling monitor election 2024-09-16T08:18:06.089 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:18:05 smithi028 ceph-mon[64115]: mon.smithi028 is new leader, mons smithi028,smithi144 in quorum (ranks 0,1) 2024-09-16T08:18:06.089 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:18:05 smithi028 ceph-mon[64115]: monmap epoch 3 2024-09-16T08:18:06.089 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:18:05 smithi028 ceph-mon[64115]: fsid 5622b2fe-7402-11ef-bceb-c7b262605968 2024-09-16T08:18:06.089 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:18:05 smithi028 ceph-mon[64115]: last_changed 2024-09-16T08:18:04.956447+0000 2024-09-16T08:18:06.089 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:18:05 smithi028 ceph-mon[64115]: created 2024-09-16T08:05:16.161732+0000 2024-09-16T08:18:06.089 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:18:05 smithi028 ceph-mon[64115]: min_mon_release 19 (squid) 2024-09-16T08:18:06.089 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:18:05 smithi028 ceph-mon[64115]: election_strategy: 1 2024-09-16T08:18:06.089 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:18:05 smithi028 ceph-mon[64115]: 0: [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] mon.smithi028 2024-09-16T08:18:06.089 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:18:05 smithi028 ceph-mon[64115]: 1: [v2:172.21.15.144:3300/0,v1:172.21.15.144:6789/0] mon.smithi144 2024-09-16T08:18:06.089 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:18:05 smithi028 ceph-mon[64115]: fsmap foofs:1 {0=foofs.smithi028.srtgev=up:active} 1 up:standby 2024-09-16T08:18:06.089 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:18:05 smithi028 ceph-mon[64115]: osdmap e72: 8 total, 8 up, 8 in 2024-09-16T08:18:06.090 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:18:05 smithi028 ceph-mon[64115]: mgrmap e42: smithi028.azvyzy(active, since 18s), standbys: smithi144.utzszj 2024-09-16T08:18:06.090 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:18:05 smithi028 ceph-mon[64115]: overall HEALTH_OK 2024-09-16T08:18:06.090 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:18:05 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:18:06.090 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:18:05 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:18:06.090 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:18:05 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-16T08:18:06.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:05 smithi144 ceph-mon[49149]: pgmap v12: 97 pgs: 97 active+clean; 338 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 23 KiB/s rd, 5.5 MiB/s wr, 550 op/s 2024-09-16T08:18:06.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:05 smithi144 ceph-mon[49149]: overall HEALTH_OK 2024-09-16T08:18:06.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:05 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "mon metadata", "id": "smithi028"} : dispatch 2024-09-16T08:18:06.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:05 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "mon metadata", "id": "smithi144"} : dispatch 2024-09-16T08:18:06.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:05 smithi144 ceph-mon[49149]: mon.smithi028 calling monitor election 2024-09-16T08:18:06.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:05 smithi144 ceph-mon[49149]: mon.smithi144 calling monitor election 2024-09-16T08:18:06.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:05 smithi144 ceph-mon[49149]: mon.smithi028 is new leader, mons smithi028,smithi144 in quorum (ranks 0,1) 2024-09-16T08:18:06.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:05 smithi144 ceph-mon[49149]: monmap epoch 3 2024-09-16T08:18:06.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:05 smithi144 ceph-mon[49149]: fsid 5622b2fe-7402-11ef-bceb-c7b262605968 2024-09-16T08:18:06.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:05 smithi144 ceph-mon[49149]: last_changed 2024-09-16T08:18:04.956447+0000 2024-09-16T08:18:06.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:05 smithi144 ceph-mon[49149]: created 2024-09-16T08:05:16.161732+0000 2024-09-16T08:18:06.333 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:05 smithi144 ceph-mon[49149]: min_mon_release 19 (squid) 2024-09-16T08:18:06.333 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:05 smithi144 ceph-mon[49149]: election_strategy: 1 2024-09-16T08:18:06.333 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:05 smithi144 ceph-mon[49149]: 0: [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] mon.smithi028 2024-09-16T08:18:06.333 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:05 smithi144 ceph-mon[49149]: 1: [v2:172.21.15.144:3300/0,v1:172.21.15.144:6789/0] mon.smithi144 2024-09-16T08:18:06.333 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:05 smithi144 ceph-mon[49149]: fsmap foofs:1 {0=foofs.smithi028.srtgev=up:active} 1 up:standby 2024-09-16T08:18:06.333 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:05 smithi144 ceph-mon[49149]: osdmap e72: 8 total, 8 up, 8 in 2024-09-16T08:18:06.333 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:05 smithi144 ceph-mon[49149]: mgrmap e42: smithi028.azvyzy(active, since 18s), standbys: smithi144.utzszj 2024-09-16T08:18:06.333 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:05 smithi144 ceph-mon[49149]: overall HEALTH_OK 2024-09-16T08:18:06.333 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:05 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:18:06.333 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:05 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:18:06.333 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:05 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-16T08:18:07.036 INFO:teuthology.orchestra.run.smithi028.stdout: 5 140735 10.44 MB/sec execute 185 sec latency 52.951 ms 2024-09-16T08:18:08.036 INFO:teuthology.orchestra.run.smithi028.stdout: 5 141248 10.47 MB/sec execute 186 sec latency 54.042 ms 2024-09-16T08:18:08.184 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:18:07 smithi028 ceph-mon[64115]: pgmap v13: 97 pgs: 97 active+clean; 338 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 5.0 KiB/s rd, 3.9 MiB/s wr, 381 op/s 2024-09-16T08:18:08.241 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:07 smithi144 ceph-mon[49149]: pgmap v13: 97 pgs: 97 active+clean; 338 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 5.0 KiB/s rd, 3.9 MiB/s wr, 381 op/s 2024-09-16T08:18:09.036 INFO:teuthology.orchestra.run.smithi028.stdout: 5 141723 10.50 MB/sec execute 187 sec latency 68.387 ms 2024-09-16T08:18:10.036 INFO:teuthology.orchestra.run.smithi028.stdout: 5 142392 10.54 MB/sec execute 188 sec latency 25.093 ms 2024-09-16T08:18:10.201 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:18:10 smithi028 ceph-mon[64115]: pgmap v14: 97 pgs: 97 active+clean; 333 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 8.9 KiB/s rd, 6.2 MiB/s wr, 631 op/s 2024-09-16T08:18:10.201 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:18:10 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:18:10.201 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:18:10 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:18:10.201 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:18:10 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:18:10.201 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:18:10 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:18:10.325 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:10 smithi144 ceph-mon[49149]: pgmap v14: 97 pgs: 97 active+clean; 333 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 8.9 KiB/s rd, 6.2 MiB/s wr, 631 op/s 2024-09-16T08:18:10.325 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:10 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:18:10.325 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:10 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:18:10.326 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:10 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:18:10.326 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:10 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:18:11.036 INFO:teuthology.orchestra.run.smithi028.stdout: 5 143011 10.56 MB/sec execute 189 sec latency 26.319 ms 2024-09-16T08:18:12.036 INFO:teuthology.orchestra.run.smithi028.stdout: 5 143709 10.58 MB/sec execute 190 sec latency 58.882 ms 2024-09-16T08:18:12.161 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:12 smithi144 ceph-mon[49149]: pgmap v15: 97 pgs: 97 active+clean; 333 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.7 MiB/s wr, 461 op/s 2024-09-16T08:18:12.232 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:18:12 smithi028 ceph-mon[64115]: pgmap v15: 97 pgs: 97 active+clean; 333 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.7 MiB/s wr, 461 op/s 2024-09-16T08:18:13.036 INFO:teuthology.orchestra.run.smithi028.stdout: 5 144300 10.62 MB/sec execute 191 sec latency 29.181 ms 2024-09-16T08:18:14.036 INFO:teuthology.orchestra.run.smithi028.stdout: 5 144864 10.65 MB/sec execute 192 sec latency 32.741 ms 2024-09-16T08:18:14.081 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:14 smithi144 ceph-mon[49149]: pgmap v16: 97 pgs: 97 active+clean; 333 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.7 MiB/s wr, 461 op/s 2024-09-16T08:18:14.081 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:14 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:18:14.082 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:14 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:18:14.082 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:14 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-16T08:18:14.082 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:14 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-16T08:18:14.082 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:14 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:18:14.082 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:14 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:18:14.082 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:14 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-09-16T08:18:14.082 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:14 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-09-16T08:18:14.082 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:14 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-16T08:18:14.246 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:18:14 smithi028 ceph-mon[64115]: pgmap v16: 97 pgs: 97 active+clean; 333 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.7 MiB/s wr, 461 op/s 2024-09-16T08:18:14.246 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:18:14 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:18:14.246 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:18:14 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:18:14.246 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:18:14 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-16T08:18:14.246 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:18:14 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-16T08:18:14.246 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:18:14 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:18:14.246 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:18:14 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:18:14.246 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:18:14 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-09-16T08:18:14.246 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:18:14 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-09-16T08:18:14.246 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:18:14 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-16T08:18:15.036 INFO:teuthology.orchestra.run.smithi028.stdout: 5 145307 10.67 MB/sec execute 193 sec latency 29.963 ms 2024-09-16T08:18:15.150 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:18:15 smithi028 ceph-mon[64115]: Reconfiguring mon.smithi028 (monmap changed)... 2024-09-16T08:18:15.150 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:18:15 smithi028 ceph-mon[64115]: Reconfiguring daemon mon.smithi028 on smithi028 2024-09-16T08:18:15.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:15 smithi144 ceph-mon[49149]: Reconfiguring mon.smithi028 (monmap changed)... 2024-09-16T08:18:15.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:15 smithi144 ceph-mon[49149]: Reconfiguring daemon mon.smithi028 on smithi028 2024-09-16T08:18:16.036 INFO:teuthology.orchestra.run.smithi028.stdout: 5 145902 10.71 MB/sec execute 194 sec latency 40.993 ms 2024-09-16T08:18:16.154 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:18:16 smithi028 ceph-mon[64115]: pgmap v17: 97 pgs: 97 active+clean; 339 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 8.5 MiB/s wr, 821 op/s 2024-09-16T08:18:16.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:16 smithi144 ceph-mon[49149]: pgmap v17: 97 pgs: 97 active+clean; 339 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 8.5 MiB/s wr, 821 op/s 2024-09-16T08:18:17.036 INFO:teuthology.orchestra.run.smithi028.stdout: 5 146317 10.71 MB/sec execute 195 sec latency 45.822 ms 2024-09-16T08:18:17.436 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:18:17 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:18:17.436 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:18:17 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:18:17.436 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:18:17 smithi028 ceph-mon[64115]: Reconfiguring mgr.smithi028.azvyzy (monmap changed)... 2024-09-16T08:18:17.436 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:18:17 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "auth get-or-create", "entity": "mgr.smithi028.azvyzy", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]} : dispatch 2024-09-16T08:18:17.436 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:18:17 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "mgr services"} : dispatch 2024-09-16T08:18:17.436 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:18:17 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-16T08:18:17.436 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:18:17 smithi028 ceph-mon[64115]: Reconfiguring daemon mgr.smithi028.azvyzy on smithi028 2024-09-16T08:18:17.436 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:18:17 smithi028 ceph-mon[64115]: pgmap v18: 97 pgs: 97 active+clean; 339 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 8.2 KiB/s rd, 6.1 MiB/s wr, 610 op/s 2024-09-16T08:18:17.437 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:18:17 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-16T08:18:17.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:17 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:18:17.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:17 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:18:17.583 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:17 smithi144 ceph-mon[49149]: Reconfiguring mgr.smithi028.azvyzy (monmap changed)... 2024-09-16T08:18:17.584 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:17 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "auth get-or-create", "entity": "mgr.smithi028.azvyzy", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]} : dispatch 2024-09-16T08:18:17.584 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:17 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "mgr services"} : dispatch 2024-09-16T08:18:17.584 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:17 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-16T08:18:17.584 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:17 smithi144 ceph-mon[49149]: Reconfiguring daemon mgr.smithi028.azvyzy on smithi028 2024-09-16T08:18:17.584 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:17 smithi144 ceph-mon[49149]: pgmap v18: 97 pgs: 97 active+clean; 339 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 8.2 KiB/s rd, 6.1 MiB/s wr, 610 op/s 2024-09-16T08:18:17.584 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:17 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-16T08:18:18.036 INFO:teuthology.orchestra.run.smithi028.stdout: 5 146585 10.67 MB/sec execute 196 sec latency 49.985 ms 2024-09-16T08:18:19.036 INFO:teuthology.orchestra.run.smithi028.stdout: 5 146850 10.65 MB/sec execute 197 sec latency 76.303 ms 2024-09-16T08:18:19.474 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:18:19 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:18:19.474 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:18:19 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:18:19.474 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:18:19 smithi028 ceph-mon[64115]: Reconfiguring crash.smithi028 (monmap changed)... 2024-09-16T08:18:19.474 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:18:19 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "auth get-or-create", "entity": "client.crash.smithi028", "caps": ["mon", "profile crash", "mgr", "profile crash"]} : dispatch 2024-09-16T08:18:19.474 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:18:19 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-16T08:18:19.474 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:18:19 smithi028 ceph-mon[64115]: Reconfiguring daemon crash.smithi028 on smithi028 2024-09-16T08:18:19.474 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:18:19 smithi028 ceph-mon[64115]: pgmap v19: 97 pgs: 97 active+clean; 337 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 13 KiB/s rd, 9.1 MiB/s wr, 892 op/s 2024-09-16T08:18:19.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:19 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:18:19.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:19 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:18:19.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:19 smithi144 ceph-mon[49149]: Reconfiguring crash.smithi028 (monmap changed)... 2024-09-16T08:18:19.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:19 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "auth get-or-create", "entity": "client.crash.smithi028", "caps": ["mon", "profile crash", "mgr", "profile crash"]} : dispatch 2024-09-16T08:18:19.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:19 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-16T08:18:19.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:19 smithi144 ceph-mon[49149]: Reconfiguring daemon crash.smithi028 on smithi028 2024-09-16T08:18:19.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:19 smithi144 ceph-mon[49149]: pgmap v19: 97 pgs: 97 active+clean; 337 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 13 KiB/s rd, 9.1 MiB/s wr, 892 op/s 2024-09-16T08:18:20.036 INFO:teuthology.orchestra.run.smithi028.stdout: 5 147169 10.63 MB/sec execute 198 sec latency 66.875 ms 2024-09-16T08:18:21.037 INFO:teuthology.orchestra.run.smithi028.stdout: 5 147491 10.61 MB/sec execute 199 sec latency 106.827 ms 2024-09-16T08:18:21.246 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:18:21 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:18:21.246 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:18:21 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:18:21.246 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:18:21 smithi028 ceph-mon[64115]: Reconfiguring osd.0 (monmap changed)... 2024-09-16T08:18:21.246 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:18:21 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "auth get", "entity": "osd.0"} : dispatch 2024-09-16T08:18:21.246 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:18:21 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-16T08:18:21.246 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:18:21 smithi028 ceph-mon[64115]: Reconfiguring daemon osd.0 on smithi028 2024-09-16T08:18:21.246 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:18:21 smithi028 ceph-mon[64115]: pgmap v20: 97 pgs: 97 active+clean; 337 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 8.8 KiB/s rd, 6.8 MiB/s wr, 642 op/s 2024-09-16T08:18:21.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:21 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:18:21.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:21 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:18:21.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:21 smithi144 ceph-mon[49149]: Reconfiguring osd.0 (monmap changed)... 2024-09-16T08:18:21.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:21 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "auth get", "entity": "osd.0"} : dispatch 2024-09-16T08:18:21.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:21 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-16T08:18:21.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:21 smithi144 ceph-mon[49149]: Reconfiguring daemon osd.0 on smithi028 2024-09-16T08:18:21.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:21 smithi144 ceph-mon[49149]: pgmap v20: 97 pgs: 97 active+clean; 337 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 8.8 KiB/s rd, 6.8 MiB/s wr, 642 op/s 2024-09-16T08:18:22.037 INFO:teuthology.orchestra.run.smithi028.stdout: 5 147993 10.64 MB/sec execute 200 sec latency 39.208 ms 2024-09-16T08:18:23.037 INFO:teuthology.orchestra.run.smithi028.stdout: 5 148537 10.66 MB/sec execute 201 sec latency 24.630 ms 2024-09-16T08:18:23.530 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:18:23 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:18:23.530 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:18:23 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:18:23.530 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:18:23 smithi028 ceph-mon[64115]: Reconfiguring osd.1 (monmap changed)... 2024-09-16T08:18:23.530 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:18:23 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "auth get", "entity": "osd.1"} : dispatch 2024-09-16T08:18:23.530 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:18:23 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-16T08:18:23.530 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:18:23 smithi028 ceph-mon[64115]: Reconfiguring daemon osd.1 on smithi028 2024-09-16T08:18:23.530 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:18:23 smithi028 ceph-mon[64115]: pgmap v21: 97 pgs: 97 active+clean; 336 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 12 KiB/s rd, 7.5 MiB/s wr, 726 op/s 2024-09-16T08:18:23.832 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:23 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:18:23.832 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:23 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:18:23.832 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:23 smithi144 ceph-mon[49149]: Reconfiguring osd.1 (monmap changed)... 2024-09-16T08:18:23.832 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:23 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "auth get", "entity": "osd.1"} : dispatch 2024-09-16T08:18:23.832 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:23 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-16T08:18:23.832 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:23 smithi144 ceph-mon[49149]: Reconfiguring daemon osd.1 on smithi028 2024-09-16T08:18:23.832 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:23 smithi144 ceph-mon[49149]: pgmap v21: 97 pgs: 97 active+clean; 336 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 12 KiB/s rd, 7.5 MiB/s wr, 726 op/s 2024-09-16T08:18:24.036 INFO:teuthology.orchestra.run.smithi028.stdout: 5 149032 10.69 MB/sec execute 202 sec latency 24.389 ms 2024-09-16T08:18:25.037 INFO:teuthology.orchestra.run.smithi028.stdout: 5 149372 10.69 MB/sec execute 203 sec latency 53.301 ms 2024-09-16T08:18:25.822 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:18:25 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:18:25.822 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:18:25 smithi028 ceph-mon[64115]: pgmap v22: 97 pgs: 97 active+clean; 331 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 12 KiB/s rd, 9.0 MiB/s wr, 866 op/s 2024-09-16T08:18:25.822 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:18:25 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:18:25.822 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:18:25 smithi028 ceph-mon[64115]: Reconfiguring osd.2 (monmap changed)... 2024-09-16T08:18:25.822 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:18:25 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "auth get", "entity": "osd.2"} : dispatch 2024-09-16T08:18:25.822 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:18:25 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-16T08:18:25.823 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:18:25 smithi028 ceph-mon[64115]: Reconfiguring daemon osd.2 on smithi028 2024-09-16T08:18:26.037 INFO:teuthology.orchestra.run.smithi028.stdout: 5 149696 10.69 MB/sec execute 204 sec latency 84.535 ms 2024-09-16T08:18:26.082 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:25 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:18:26.082 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:25 smithi144 ceph-mon[49149]: pgmap v22: 97 pgs: 97 active+clean; 331 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 12 KiB/s rd, 9.0 MiB/s wr, 866 op/s 2024-09-16T08:18:26.082 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:25 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:18:26.082 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:25 smithi144 ceph-mon[49149]: Reconfiguring osd.2 (monmap changed)... 2024-09-16T08:18:26.082 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:25 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "auth get", "entity": "osd.2"} : dispatch 2024-09-16T08:18:26.082 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:25 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-16T08:18:26.082 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:25 smithi144 ceph-mon[49149]: Reconfiguring daemon osd.2 on smithi028 2024-09-16T08:18:27.037 INFO:teuthology.orchestra.run.smithi028.stdout: 5 150131 10.67 MB/sec execute 205 sec latency 29.698 ms 2024-09-16T08:18:27.844 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:18:27 smithi028 ceph-mon[64115]: pgmap v23: 97 pgs: 97 active+clean; 331 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 8.1 KiB/s rd, 5.2 MiB/s wr, 507 op/s 2024-09-16T08:18:27.844 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:18:27 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:18:27.844 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:18:27 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:18:27.844 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:18:27 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "auth get", "entity": "osd.3"} : dispatch 2024-09-16T08:18:27.844 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:18:27 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-16T08:18:28.037 INFO:teuthology.orchestra.run.smithi028.stdout: 5 150496 10.66 MB/sec execute 206 sec latency 32.044 ms 2024-09-16T08:18:28.081 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:27 smithi144 ceph-mon[49149]: pgmap v23: 97 pgs: 97 active+clean; 331 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 8.1 KiB/s rd, 5.2 MiB/s wr, 507 op/s 2024-09-16T08:18:28.081 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:27 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:18:28.081 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:27 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:18:28.082 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:27 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "auth get", "entity": "osd.3"} : dispatch 2024-09-16T08:18:28.082 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:27 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-16T08:18:28.678 INFO:teuthology.orchestra.run.smithi028.stdout:true 2024-09-16T08:18:28.858 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:18:28 smithi028 ceph-mon[64115]: Reconfiguring osd.3 (monmap changed)... 2024-09-16T08:18:28.858 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:18:28 smithi028 ceph-mon[64115]: Reconfiguring daemon osd.3 on smithi028 2024-09-16T08:18:29.037 INFO:teuthology.orchestra.run.smithi028.stdout: 5 150933 10.65 MB/sec execute 207 sec latency 41.329 ms 2024-09-16T08:18:29.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:28 smithi144 ceph-mon[49149]: Reconfiguring osd.3 (monmap changed)... 2024-09-16T08:18:29.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:28 smithi144 ceph-mon[49149]: Reconfiguring daemon osd.3 on smithi028 2024-09-16T08:18:29.853 INFO:teuthology.orchestra.run.smithi028.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-16T08:18:29.853 INFO:teuthology.orchestra.run.smithi028.stdout:alertmanager.smithi028 smithi028 *:9093,9094 running (2m) 34s ago 11m 19.7M - 0.27.0 11f11916f8cd a14fcb4cb2b6 2024-09-16T08:18:29.853 INFO:teuthology.orchestra.run.smithi028.stdout:crash.smithi028 smithi028 running (11m) 34s ago 11m 6405k - 17.2.0 e1d6a67b021e 69e795c000fd 2024-09-16T08:18:29.853 INFO:teuthology.orchestra.run.smithi028.stdout:crash.smithi144 smithi144 running (11m) 20s ago 11m 6400k - 17.2.0 e1d6a67b021e bbe54b51649c 2024-09-16T08:18:29.853 INFO:teuthology.orchestra.run.smithi028.stdout:grafana.smithi028 smithi028 *:3000 running (2m) 34s ago 11m 43.8M - dad864ee21e9 b55ba8b944a1 2024-09-16T08:18:29.853 INFO:teuthology.orchestra.run.smithi028.stdout:mds.foofs.smithi028.srtgev smithi028 running (6m) 34s ago 6m 37.1M - 17.2.0 e1d6a67b021e 250e7a7f754c 2024-09-16T08:18:29.853 INFO:teuthology.orchestra.run.smithi028.stdout:mds.foofs.smithi144.oozgsi smithi144 running (6m) 20s ago 6m 16.5M - 17.2.0 e1d6a67b021e ecf4c2bb0fdc 2024-09-16T08:18:29.854 INFO:teuthology.orchestra.run.smithi028.stdout:mgr.smithi028.azvyzy smithi028 *:8443,9283,8765 running (116s) 34s ago 13m 488M - 19.3.0-4917-g8293d73f 425e316ba8eb e01d4382d03f 2024-09-16T08:18:29.854 INFO:teuthology.orchestra.run.smithi028.stdout:mgr.smithi144.utzszj smithi144 *:8443,9283,8765 running (68s) 20s ago 11m 452M - 19.3.0-4917-g8293d73f 425e316ba8eb 71f7f802bffb 2024-09-16T08:18:29.854 INFO:teuthology.orchestra.run.smithi028.stdout:mon.smithi028 smithi028 running (50s) 34s ago 13m 31.9M 2048M 19.3.0-4917-g8293d73f 425e316ba8eb 14efcedc0088 2024-09-16T08:18:29.854 INFO:teuthology.orchestra.run.smithi028.stdout:mon.smithi144 smithi144 running (26s) 20s ago 11m 26.2M 2048M 19.3.0-4917-g8293d73f 425e316ba8eb c081c5ed44b5 2024-09-16T08:18:29.854 INFO:teuthology.orchestra.run.smithi028.stdout:nfs.foo.0.0.smithi028.dkttgj smithi028 *:2049 running (5m) 34s ago 5m 274M - 4.0 e1d6a67b021e b5dccc45a6ed 2024-09-16T08:18:29.854 INFO:teuthology.orchestra.run.smithi028.stdout:nfs.foo.1.0.smithi144.epipkg smithi144 *:2049 running (5m) 20s ago 5m 86.9M - 4.0 e1d6a67b021e faa89e61f50e 2024-09-16T08:18:29.854 INFO:teuthology.orchestra.run.smithi028.stdout:node-exporter.smithi028 smithi028 *:9100 running (2m) 34s ago 11m 9977k - 1.7.0 72c9c2088986 9e2b8e8a63a9 2024-09-16T08:18:29.854 INFO:teuthology.orchestra.run.smithi028.stdout:node-exporter.smithi144 smithi144 *:9100 running (2m) 20s ago 11m 11.1M - 1.7.0 72c9c2088986 78834c5e992f 2024-09-16T08:18:29.854 INFO:teuthology.orchestra.run.smithi028.stdout:osd.0 smithi028 running (10m) 34s ago 10m 485M 1937M 17.2.0 e1d6a67b021e 8184fbc3871f 2024-09-16T08:18:29.854 INFO:teuthology.orchestra.run.smithi028.stdout:osd.1 smithi028 running (9m) 34s ago 9m 408M 1937M 17.2.0 e1d6a67b021e 6911d5c68d54 2024-09-16T08:18:29.854 INFO:teuthology.orchestra.run.smithi028.stdout:osd.2 smithi028 running (9m) 34s ago 9m 299M 1937M 17.2.0 e1d6a67b021e 90b27d8375a2 2024-09-16T08:18:29.854 INFO:teuthology.orchestra.run.smithi028.stdout:osd.3 smithi028 running (9m) 34s ago 9m 448M 1937M 17.2.0 e1d6a67b021e dac477d8b04f 2024-09-16T08:18:29.854 INFO:teuthology.orchestra.run.smithi028.stdout:osd.4 smithi144 running (8m) 20s ago 8m 477M 2704M 17.2.0 e1d6a67b021e 86719c60975c 2024-09-16T08:18:29.854 INFO:teuthology.orchestra.run.smithi028.stdout:osd.5 smithi144 running (8m) 20s ago 8m 536M 2704M 17.2.0 e1d6a67b021e 70452c145c60 2024-09-16T08:18:29.855 INFO:teuthology.orchestra.run.smithi028.stdout:osd.6 smithi144 running (7m) 20s ago 7m 455M 2704M 17.2.0 e1d6a67b021e 7f9981be792f 2024-09-16T08:18:29.855 INFO:teuthology.orchestra.run.smithi028.stdout:osd.7 smithi144 running (7m) 20s ago 7m 350M 2704M 17.2.0 e1d6a67b021e dada74ddbe4c 2024-09-16T08:18:29.855 INFO:teuthology.orchestra.run.smithi028.stdout:prometheus.smithi028 smithi028 *:9095 running (74s) 34s ago 10m 49.5M - 2.51.0 1d3b7f56885b 42667809df2b 2024-09-16T08:18:29.864 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:18:29 smithi028 ceph-mon[64115]: from='client.44101 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T08:18:29.864 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:18:29 smithi028 ceph-mon[64115]: pgmap v24: 97 pgs: 97 active+clean; 334 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 12 KiB/s rd, 7.7 MiB/s wr, 759 op/s 2024-09-16T08:18:29.864 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:18:29 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:18:29.864 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:18:29 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:18:29.864 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:18:29 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "auth get-or-create", "entity": "mds.foofs.smithi028.srtgev", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-09-16T08:18:29.864 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:18:29 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-16T08:18:30.037 INFO:teuthology.orchestra.run.smithi028.stdout: 5 151217 10.64 MB/sec execute 208 sec latency 65.785 ms 2024-09-16T08:18:30.273 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:29 smithi144 ceph-mon[49149]: from='client.44101 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T08:18:30.273 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:29 smithi144 ceph-mon[49149]: pgmap v24: 97 pgs: 97 active+clean; 334 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 12 KiB/s rd, 7.7 MiB/s wr, 759 op/s 2024-09-16T08:18:30.273 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:29 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:18:30.273 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:29 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:18:30.273 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:29 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "auth get-or-create", "entity": "mds.foofs.smithi028.srtgev", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-09-16T08:18:30.273 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:29 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-16T08:18:30.476 INFO:teuthology.orchestra.run.smithi028.stdout:{ 2024-09-16T08:18:30.476 INFO:teuthology.orchestra.run.smithi028.stdout: "mon": { 2024-09-16T08:18:30.476 INFO:teuthology.orchestra.run.smithi028.stdout: "ceph version 19.3.0-4917-g8293d73f (8293d73f8690540e843a81caec373f9cc29cf705) squid (dev)": 2 2024-09-16T08:18:30.476 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-09-16T08:18:30.476 INFO:teuthology.orchestra.run.smithi028.stdout: "mgr": { 2024-09-16T08:18:30.476 INFO:teuthology.orchestra.run.smithi028.stdout: "ceph version 19.3.0-4917-g8293d73f (8293d73f8690540e843a81caec373f9cc29cf705) squid (dev)": 2 2024-09-16T08:18:30.476 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-09-16T08:18:30.477 INFO:teuthology.orchestra.run.smithi028.stdout: "osd": { 2024-09-16T08:18:30.477 INFO:teuthology.orchestra.run.smithi028.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-09-16T08:18:30.477 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-09-16T08:18:30.477 INFO:teuthology.orchestra.run.smithi028.stdout: "mds": { 2024-09-16T08:18:30.477 INFO:teuthology.orchestra.run.smithi028.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-16T08:18:30.477 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-09-16T08:18:30.477 INFO:teuthology.orchestra.run.smithi028.stdout: "overall": { 2024-09-16T08:18:30.477 INFO:teuthology.orchestra.run.smithi028.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 10, 2024-09-16T08:18:30.477 INFO:teuthology.orchestra.run.smithi028.stdout: "ceph version 19.3.0-4917-g8293d73f (8293d73f8690540e843a81caec373f9cc29cf705) squid (dev)": 4 2024-09-16T08:18:30.477 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-09-16T08:18:30.477 INFO:teuthology.orchestra.run.smithi028.stdout:} 2024-09-16T08:18:30.906 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:18:30 smithi028 ceph-mon[64115]: from='client.44105 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T08:18:30.906 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:18:30 smithi028 ceph-mon[64115]: Reconfiguring mds.foofs.smithi028.srtgev (monmap changed)... 2024-09-16T08:18:30.906 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:18:30 smithi028 ceph-mon[64115]: Reconfiguring daemon mds.foofs.smithi028.srtgev on smithi028 2024-09-16T08:18:30.906 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:18:30 smithi028 ceph-mon[64115]: from='client.34164 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T08:18:30.906 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:18:30 smithi028 ceph-mon[64115]: from='client.? 172.21.15.28:0/2792910594' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-16T08:18:31.035 INFO:teuthology.orchestra.run.smithi028.stdout:{ 2024-09-16T08:18:31.035 INFO:teuthology.orchestra.run.smithi028.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:f7ad55f9ccd17ba66aadb19b4d291e67b60ea24e5d8a0c0e92941fd59c81e4ce", 2024-09-16T08:18:31.035 INFO:teuthology.orchestra.run.smithi028.stdout: "in_progress": true, 2024-09-16T08:18:31.036 INFO:teuthology.orchestra.run.smithi028.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-16T08:18:31.036 INFO:teuthology.orchestra.run.smithi028.stdout: "services_complete": [ 2024-09-16T08:18:31.036 INFO:teuthology.orchestra.run.smithi028.stdout: "mon", 2024-09-16T08:18:31.036 INFO:teuthology.orchestra.run.smithi028.stdout: "mgr" 2024-09-16T08:18:31.036 INFO:teuthology.orchestra.run.smithi028.stdout: ], 2024-09-16T08:18:31.036 INFO:teuthology.orchestra.run.smithi028.stdout: "progress": "4/23 daemons upgraded", 2024-09-16T08:18:31.036 INFO:teuthology.orchestra.run.smithi028.stdout: "message": "Currently upgrading mon daemons", 2024-09-16T08:18:31.036 INFO:teuthology.orchestra.run.smithi028.stdout: "is_paused": false 2024-09-16T08:18:31.036 INFO:teuthology.orchestra.run.smithi028.stdout:} 2024-09-16T08:18:31.037 INFO:teuthology.orchestra.run.smithi028.stdout: 5 151669 10.67 MB/sec execute 209 sec latency 51.675 ms 2024-09-16T08:18:31.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:30 smithi144 ceph-mon[49149]: from='client.44105 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T08:18:31.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:30 smithi144 ceph-mon[49149]: Reconfiguring mds.foofs.smithi028.srtgev (monmap changed)... 2024-09-16T08:18:31.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:30 smithi144 ceph-mon[49149]: Reconfiguring daemon mds.foofs.smithi028.srtgev on smithi028 2024-09-16T08:18:31.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:30 smithi144 ceph-mon[49149]: from='client.34164 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T08:18:31.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:30 smithi144 ceph-mon[49149]: from='client.? 172.21.15.28:0/2792910594' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-16T08:18:31.672 INFO:teuthology.orchestra.run.smithi028.stdout:HEALTH_OK 2024-09-16T08:18:31.917 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:18:31 smithi028 ceph-mon[64115]: pgmap v25: 97 pgs: 97 active+clean; 334 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 7.3 KiB/s rd, 4.7 MiB/s wr, 477 op/s 2024-09-16T08:18:31.917 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:18:31 smithi028 ceph-mon[64115]: from='client.34172 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T08:18:31.917 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:18:31 smithi028 ceph-mon[64115]: from='client.? 172.21.15.28:0/3757691552' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-16T08:18:32.037 INFO:teuthology.orchestra.run.smithi028.stdout: 5 152003 10.66 MB/sec execute 210 sec latency 41.968 ms 2024-09-16T08:18:32.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:31 smithi144 ceph-mon[49149]: pgmap v25: 97 pgs: 97 active+clean; 334 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 7.3 KiB/s rd, 4.7 MiB/s wr, 477 op/s 2024-09-16T08:18:32.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:31 smithi144 ceph-mon[49149]: from='client.34172 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T08:18:32.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:31 smithi144 ceph-mon[49149]: from='client.? 172.21.15.28:0/3757691552' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-16T08:18:33.037 INFO:teuthology.orchestra.run.smithi028.stdout: 5 152787 10.74 MB/sec execute 211 sec latency 24.593 ms 2024-09-16T08:18:33.174 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:18:33 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:18:33.174 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:18:33 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:18:33.174 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:18:33 smithi028 ceph-mon[64115]: Reconfiguring crash.smithi144 (monmap changed)... 2024-09-16T08:18:33.174 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:18:33 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "auth get-or-create", "entity": "client.crash.smithi144", "caps": ["mon", "profile crash", "mgr", "profile crash"]} : dispatch 2024-09-16T08:18:33.174 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:18:33 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-16T08:18:33.174 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:18:33 smithi028 ceph-mon[64115]: Reconfiguring daemon crash.smithi144 on smithi144 2024-09-16T08:18:33.174 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:18:33 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-16T08:18:33.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:33 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:18:33.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:33 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:18:33.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:33 smithi144 ceph-mon[49149]: Reconfiguring crash.smithi144 (monmap changed)... 2024-09-16T08:18:33.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:33 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "auth get-or-create", "entity": "client.crash.smithi144", "caps": ["mon", "profile crash", "mgr", "profile crash"]} : dispatch 2024-09-16T08:18:33.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:33 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-16T08:18:33.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:33 smithi144 ceph-mon[49149]: Reconfiguring daemon crash.smithi144 on smithi144 2024-09-16T08:18:33.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:33 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-16T08:18:34.037 INFO:teuthology.orchestra.run.smithi028.stdout: 5 153379 10.74 MB/sec execute 212 sec latency 31.643 ms 2024-09-16T08:18:34.180 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:18:34 smithi028 ceph-mon[64115]: pgmap v26: 97 pgs: 97 active+clean; 330 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 5.3 MiB/s wr, 543 op/s 2024-09-16T08:18:34.181 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:18:34 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:18:34.181 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:18:34 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:18:34.181 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:18:34 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "auth get-or-create", "entity": "mgr.smithi144.utzszj", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]} : dispatch 2024-09-16T08:18:34.181 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:18:34 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "mgr services"} : dispatch 2024-09-16T08:18:34.181 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:18:34 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-16T08:18:34.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:34 smithi144 ceph-mon[49149]: pgmap v26: 97 pgs: 97 active+clean; 330 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 5.3 MiB/s wr, 543 op/s 2024-09-16T08:18:34.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:34 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:18:34.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:34 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:18:34.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:34 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "auth get-or-create", "entity": "mgr.smithi144.utzszj", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]} : dispatch 2024-09-16T08:18:34.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:34 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "mgr services"} : dispatch 2024-09-16T08:18:34.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:34 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-16T08:18:35.037 INFO:teuthology.orchestra.run.smithi028.stdout: 5 153816 10.74 MB/sec execute 213 sec latency 39.004 ms 2024-09-16T08:18:35.189 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:18:35 smithi028 ceph-mon[64115]: Reconfiguring mgr.smithi144.utzszj (monmap changed)... 2024-09-16T08:18:35.190 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:18:35 smithi028 ceph-mon[64115]: Reconfiguring daemon mgr.smithi144.utzszj on smithi144 2024-09-16T08:18:35.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:35 smithi144 ceph-mon[49149]: Reconfiguring mgr.smithi144.utzszj (monmap changed)... 2024-09-16T08:18:35.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:35 smithi144 ceph-mon[49149]: Reconfiguring daemon mgr.smithi144.utzszj on smithi144 2024-09-16T08:18:36.037 INFO:teuthology.orchestra.run.smithi028.stdout: 5 154306 10.74 MB/sec execute 214 sec latency 36.447 ms 2024-09-16T08:18:36.081 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:36 smithi144 ceph-mon[49149]: pgmap v27: 97 pgs: 97 active+clean; 337 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.8 KiB/s rd, 6.8 MiB/s wr, 649 op/s 2024-09-16T08:18:36.081 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:36 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:18:36.081 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:36 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:18:36.082 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:36 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-09-16T08:18:36.082 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:36 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-09-16T08:18:36.082 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:36 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-16T08:18:36.205 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:18:36 smithi028 ceph-mon[64115]: pgmap v27: 97 pgs: 97 active+clean; 337 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.8 KiB/s rd, 6.8 MiB/s wr, 649 op/s 2024-09-16T08:18:36.205 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:18:36 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:18:36.205 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:18:36 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:18:36.205 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:18:36 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-09-16T08:18:36.205 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:18:36 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-09-16T08:18:36.205 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:18:36 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-16T08:18:37.037 INFO:teuthology.orchestra.run.smithi028.stdout: 5 154736 10.73 MB/sec execute 215 sec latency 31.204 ms 2024-09-16T08:18:37.221 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:18:37 smithi028 ceph-mon[64115]: Reconfiguring mon.smithi144 (monmap changed)... 2024-09-16T08:18:37.221 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:18:37 smithi028 ceph-mon[64115]: Reconfiguring daemon mon.smithi144 on smithi144 2024-09-16T08:18:37.221 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:18:37 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:18:37.221 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:18:37 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:18:37.221 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:18:37 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "auth get", "entity": "osd.4"} : dispatch 2024-09-16T08:18:37.221 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:18:37 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-16T08:18:37.267 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:37 smithi144 ceph-mon[49149]: Reconfiguring mon.smithi144 (monmap changed)... 2024-09-16T08:18:37.268 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:37 smithi144 ceph-mon[49149]: Reconfiguring daemon mon.smithi144 on smithi144 2024-09-16T08:18:37.268 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:37 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:18:37.268 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:37 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:18:37.268 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:37 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "auth get", "entity": "osd.4"} : dispatch 2024-09-16T08:18:37.268 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:37 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-16T08:18:38.037 INFO:teuthology.orchestra.run.smithi028.stdout: 5 155131 10.75 MB/sec execute 216 sec latency 40.022 ms 2024-09-16T08:18:38.235 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:18:38 smithi028 ceph-mon[64115]: pgmap v28: 97 pgs: 97 active+clean; 337 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.8 KiB/s rd, 5.3 MiB/s wr, 509 op/s 2024-09-16T08:18:38.235 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:18:38 smithi028 ceph-mon[64115]: Reconfiguring osd.4 (monmap changed)... 2024-09-16T08:18:38.235 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:18:38 smithi028 ceph-mon[64115]: Reconfiguring daemon osd.4 on smithi144 2024-09-16T08:18:38.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:38 smithi144 ceph-mon[49149]: pgmap v28: 97 pgs: 97 active+clean; 337 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.8 KiB/s rd, 5.3 MiB/s wr, 509 op/s 2024-09-16T08:18:38.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:38 smithi144 ceph-mon[49149]: Reconfiguring osd.4 (monmap changed)... 2024-09-16T08:18:38.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:38 smithi144 ceph-mon[49149]: Reconfiguring daemon osd.4 on smithi144 2024-09-16T08:18:39.037 INFO:teuthology.orchestra.run.smithi028.stdout: 5 155406 10.73 MB/sec execute 217 sec latency 61.964 ms 2024-09-16T08:18:39.800 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:18:39 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:18:39.800 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:18:39 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:18:39.800 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:18:39 smithi028 ceph-mon[64115]: Reconfiguring osd.5 (monmap changed)... 2024-09-16T08:18:39.800 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:18:39 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "auth get", "entity": "osd.5"} : dispatch 2024-09-16T08:18:39.800 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:18:39 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-16T08:18:39.801 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:18:39 smithi028 ceph-mon[64115]: Reconfiguring daemon osd.5 on smithi144 2024-09-16T08:18:39.801 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:18:39 smithi028 ceph-mon[64115]: pgmap v29: 97 pgs: 97 active+clean; 328 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 12 KiB/s rd, 8.1 MiB/s wr, 805 op/s 2024-09-16T08:18:39.831 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:39 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:18:39.831 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:39 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:18:39.831 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:39 smithi144 ceph-mon[49149]: Reconfiguring osd.5 (monmap changed)... 2024-09-16T08:18:39.831 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:39 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "auth get", "entity": "osd.5"} : dispatch 2024-09-16T08:18:39.832 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:39 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-16T08:18:39.832 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:39 smithi144 ceph-mon[49149]: Reconfiguring daemon osd.5 on smithi144 2024-09-16T08:18:39.832 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:39 smithi144 ceph-mon[49149]: pgmap v29: 97 pgs: 97 active+clean; 328 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 12 KiB/s rd, 8.1 MiB/s wr, 805 op/s 2024-09-16T08:18:40.037 INFO:teuthology.orchestra.run.smithi028.stdout: 5 155754 10.73 MB/sec execute 218 sec latency 73.986 ms 2024-09-16T08:18:41.037 INFO:teuthology.orchestra.run.smithi028.stdout: 5 156051 10.72 MB/sec execute 219 sec latency 76.687 ms 2024-09-16T08:18:41.938 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:18:41 smithi028 ceph-mon[64115]: pgmap v30: 97 pgs: 97 active+clean; 328 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 8.1 KiB/s rd, 5.5 MiB/s wr, 553 op/s 2024-09-16T08:18:41.938 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:18:41 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:18:41.938 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:18:41 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:18:41.938 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:18:41 smithi028 ceph-mon[64115]: Reconfiguring osd.6 (monmap changed)... 2024-09-16T08:18:41.938 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:18:41 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "auth get", "entity": "osd.6"} : dispatch 2024-09-16T08:18:41.939 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:18:41 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-16T08:18:41.939 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:18:41 smithi028 ceph-mon[64115]: Reconfiguring daemon osd.6 on smithi144 2024-09-16T08:18:42.037 INFO:teuthology.orchestra.run.smithi028.stdout: 5 156294 10.73 MB/sec execute 220 sec latency 90.930 ms 2024-09-16T08:18:42.266 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:41 smithi144 ceph-mon[49149]: pgmap v30: 97 pgs: 97 active+clean; 328 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 8.1 KiB/s rd, 5.5 MiB/s wr, 553 op/s 2024-09-16T08:18:42.266 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:41 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:18:42.267 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:41 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:18:42.267 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:41 smithi144 ceph-mon[49149]: Reconfiguring osd.6 (monmap changed)... 2024-09-16T08:18:42.267 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:41 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "auth get", "entity": "osd.6"} : dispatch 2024-09-16T08:18:42.267 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:41 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-16T08:18:42.267 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:41 smithi144 ceph-mon[49149]: Reconfiguring daemon osd.6 on smithi144 2024-09-16T08:18:43.037 INFO:teuthology.orchestra.run.smithi028.stdout: 5 156520 10.71 MB/sec execute 221 sec latency 50.874 ms 2024-09-16T08:18:43.423 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:18:43 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:18:43.423 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:18:43 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:18:43.423 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:18:43 smithi028 ceph-mon[64115]: Reconfiguring osd.7 (monmap changed)... 2024-09-16T08:18:43.423 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:18:43 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "auth get", "entity": "osd.7"} : dispatch 2024-09-16T08:18:43.423 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:18:43 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-16T08:18:43.423 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:18:43 smithi028 ceph-mon[64115]: Reconfiguring daemon osd.7 on smithi144 2024-09-16T08:18:43.424 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:18:43 smithi028 ceph-mon[64115]: pgmap v31: 97 pgs: 97 active+clean; 334 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 6.4 MiB/s wr, 677 op/s 2024-09-16T08:18:43.831 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:43 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:18:43.832 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:43 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:18:43.832 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:43 smithi144 ceph-mon[49149]: Reconfiguring osd.7 (monmap changed)... 2024-09-16T08:18:43.832 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:43 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "auth get", "entity": "osd.7"} : dispatch 2024-09-16T08:18:43.832 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:43 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-16T08:18:43.832 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:43 smithi144 ceph-mon[49149]: Reconfiguring daemon osd.7 on smithi144 2024-09-16T08:18:43.832 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:43 smithi144 ceph-mon[49149]: pgmap v31: 97 pgs: 97 active+clean; 334 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 6.4 MiB/s wr, 677 op/s 2024-09-16T08:18:44.037 INFO:teuthology.orchestra.run.smithi028.stdout: 5 156775 10.68 MB/sec execute 222 sec latency 99.686 ms 2024-09-16T08:18:45.037 INFO:teuthology.orchestra.run.smithi028.stdout: 5 157068 10.65 MB/sec execute 223 sec latency 45.854 ms 2024-09-16T08:18:45.141 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:18:44 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:18:45.141 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:18:44 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:18:45.141 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:18:44 smithi028 ceph-mon[64115]: Reconfiguring mds.foofs.smithi144.oozgsi (monmap changed)... 2024-09-16T08:18:45.141 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:18:44 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "auth get-or-create", "entity": "mds.foofs.smithi144.oozgsi", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-09-16T08:18:45.141 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:18:44 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-16T08:18:45.141 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:18:44 smithi028 ceph-mon[64115]: Reconfiguring daemon mds.foofs.smithi144.oozgsi on smithi144 2024-09-16T08:18:45.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:44 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:18:45.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:44 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:18:45.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:44 smithi144 ceph-mon[49149]: Reconfiguring mds.foofs.smithi144.oozgsi (monmap changed)... 2024-09-16T08:18:45.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:44 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "auth get-or-create", "entity": "mds.foofs.smithi144.oozgsi", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-09-16T08:18:45.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:44 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-16T08:18:45.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:44 smithi144 ceph-mon[49149]: Reconfiguring daemon mds.foofs.smithi144.oozgsi on smithi144 2024-09-16T08:18:46.037 INFO:teuthology.orchestra.run.smithi028.stdout: 5 157393 10.64 MB/sec execute 224 sec latency 46.942 ms 2024-09-16T08:18:46.147 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:18:45 smithi028 ceph-mon[64115]: pgmap v32: 97 pgs: 97 active+clean; 335 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.6 KiB/s rd, 7.0 MiB/s wr, 665 op/s 2024-09-16T08:18:46.147 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:18:45 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:18:46.147 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:18:45 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:18:46.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:45 smithi144 ceph-mon[49149]: pgmap v32: 97 pgs: 97 active+clean; 335 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.6 KiB/s rd, 7.0 MiB/s wr, 665 op/s 2024-09-16T08:18:46.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:45 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:18:46.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:45 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:18:47.037 INFO:teuthology.orchestra.run.smithi028.stdout: 5 157833 10.64 MB/sec execute 225 sec latency 39.199 ms 2024-09-16T08:18:47.168 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:18:46 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-16T08:18:47.168 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:18:46 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "versions"} : dispatch 2024-09-16T08:18:47.168 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:18:46 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "versions"} : dispatch 2024-09-16T08:18:47.168 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:18:46 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:18:47.168 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:18:46 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.smithi028"} : dispatch 2024-09-16T08:18:47.168 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:18:46 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.smithi028"}]': finished 2024-09-16T08:18:47.168 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:18:46 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.smithi144"} : dispatch 2024-09-16T08:18:47.169 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:18:46 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.smithi144"}]': finished 2024-09-16T08:18:47.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:46 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-16T08:18:47.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:46 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "versions"} : dispatch 2024-09-16T08:18:47.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:46 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "versions"} : dispatch 2024-09-16T08:18:47.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:46 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:18:47.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:46 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.smithi028"} : dispatch 2024-09-16T08:18:47.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:46 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.smithi028"}]': finished 2024-09-16T08:18:47.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:46 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.smithi144"} : dispatch 2024-09-16T08:18:47.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:46 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.smithi144"}]': finished 2024-09-16T08:18:48.037 INFO:teuthology.orchestra.run.smithi028.stdout: 5 158295 10.65 MB/sec execute 226 sec latency 50.944 ms 2024-09-16T08:18:48.194 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:18:47 smithi028 ceph-mon[64115]: Upgrade: Setting container_image for all mon 2024-09-16T08:18:48.194 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:18:47 smithi028 ceph-mon[64115]: pgmap v33: 97 pgs: 97 active+clean; 335 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.6 KiB/s rd, 4.8 MiB/s wr, 474 op/s 2024-09-16T08:18:48.194 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:18:47 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-16T08:18:48.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:47 smithi144 ceph-mon[49149]: Upgrade: Setting container_image for all mon 2024-09-16T08:18:48.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:47 smithi144 ceph-mon[49149]: pgmap v33: 97 pgs: 97 active+clean; 335 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.6 KiB/s rd, 4.8 MiB/s wr, 474 op/s 2024-09-16T08:18:48.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:47 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-16T08:18:49.037 INFO:teuthology.orchestra.run.smithi028.stdout: 5 158823 10.67 MB/sec execute 227 sec latency 41.222 ms 2024-09-16T08:18:49.958 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:18:49 smithi028 ceph-mon[64115]: Upgrade: Updating crash.smithi028 (1/2) 2024-09-16T08:18:49.958 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:18:49 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:18:49.958 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:18:49 smithi028 ceph-mon[64115]: pgmap v34: 97 pgs: 97 active+clean; 333 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 6.7 MiB/s wr, 681 op/s 2024-09-16T08:18:49.958 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:18:49 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "auth get-or-create", "entity": "client.crash.smithi028", "caps": ["mon", "profile crash", "mgr", "profile crash"]} : dispatch 2024-09-16T08:18:49.958 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:18:49 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-16T08:18:49.958 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:18:49 smithi028 ceph-mon[64115]: Deploying daemon crash.smithi028 on smithi028 2024-09-16T08:18:50.037 INFO:teuthology.orchestra.run.smithi028.stdout: 5 159485 10.71 MB/sec execute 228 sec latency 38.011 ms 2024-09-16T08:18:50.081 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:49 smithi144 ceph-mon[49149]: Upgrade: Updating crash.smithi028 (1/2) 2024-09-16T08:18:50.082 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:49 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:18:50.082 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:49 smithi144 ceph-mon[49149]: pgmap v34: 97 pgs: 97 active+clean; 333 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 6.7 MiB/s wr, 681 op/s 2024-09-16T08:18:50.082 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:49 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "auth get-or-create", "entity": "client.crash.smithi028", "caps": ["mon", "profile crash", "mgr", "profile crash"]} : dispatch 2024-09-16T08:18:50.082 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:49 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-16T08:18:50.082 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:49 smithi144 ceph-mon[49149]: Deploying daemon crash.smithi028 on smithi028 2024-09-16T08:18:51.038 INFO:teuthology.orchestra.run.smithi028.stdout: 5 159936 10.73 MB/sec execute 229 sec latency 67.478 ms 2024-09-16T08:18:51.985 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:18:51 smithi028 ceph-mon[64115]: pgmap v35: 97 pgs: 97 active+clean; 333 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 4.0 MiB/s wr, 385 op/s 2024-09-16T08:18:52.038 INFO:teuthology.orchestra.run.smithi028.stdout: 5 160335 10.72 MB/sec execute 230 sec latency 82.409 ms 2024-09-16T08:18:52.081 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:51 smithi144 ceph-mon[49149]: pgmap v35: 97 pgs: 97 active+clean; 333 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 4.0 MiB/s wr, 385 op/s 2024-09-16T08:18:53.038 INFO:teuthology.orchestra.run.smithi028.stdout: 5 160677 10.71 MB/sec execute 231 sec latency 82.440 ms 2024-09-16T08:18:54.023 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:18:53 smithi028 ceph-mon[64115]: pgmap v36: 97 pgs: 97 active+clean; 333 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 4.5 MiB/s wr, 407 op/s 2024-09-16T08:18:54.037 INFO:teuthology.orchestra.run.smithi028.stdout: 5 161054 10.71 MB/sec execute 232 sec latency 41.288 ms 2024-09-16T08:18:54.086 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:53 smithi144 ceph-mon[49149]: pgmap v36: 97 pgs: 97 active+clean; 333 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 4.5 MiB/s wr, 407 op/s 2024-09-16T08:18:55.038 INFO:teuthology.orchestra.run.smithi028.stdout: 5 161439 10.69 MB/sec execute 233 sec latency 53.374 ms 2024-09-16T08:18:55.794 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:55 smithi144 ceph-mon[49149]: pgmap v37: 97 pgs: 97 active+clean; 333 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 7.7 KiB/s rd, 6.2 MiB/s wr, 557 op/s 2024-09-16T08:18:55.794 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:55 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:18:55.794 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:55 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:18:55.800 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:18:55 smithi028 ceph-mon[64115]: pgmap v37: 97 pgs: 97 active+clean; 333 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 7.7 KiB/s rd, 6.2 MiB/s wr, 557 op/s 2024-09-16T08:18:55.800 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:18:55 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:18:55.800 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:18:55 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:18:56.038 INFO:teuthology.orchestra.run.smithi028.stdout: 5 161824 10.69 MB/sec execute 234 sec latency 44.480 ms 2024-09-16T08:18:57.038 INFO:teuthology.orchestra.run.smithi028.stdout: 5 162153 10.68 MB/sec execute 235 sec latency 51.046 ms 2024-09-16T08:18:58.038 INFO:teuthology.orchestra.run.smithi028.stdout: 5 162476 10.68 MB/sec execute 236 sec latency 64.221 ms 2024-09-16T08:18:58.134 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:18:57 smithi028 ceph-mon[64115]: pgmap v38: 97 pgs: 97 active+clean; 333 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 7.7 KiB/s rd, 5.0 MiB/s wr, 503 op/s 2024-09-16T08:18:58.134 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:18:57 smithi028 ceph-mon[64115]: Upgrade: Updating crash.smithi144 (2/2) 2024-09-16T08:18:58.134 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:18:57 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:18:58.134 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:18:57 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "auth get-or-create", "entity": "client.crash.smithi144", "caps": ["mon", "profile crash", "mgr", "profile crash"]} : dispatch 2024-09-16T08:18:58.134 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:18:57 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-16T08:18:58.134 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:18:57 smithi028 ceph-mon[64115]: Deploying daemon crash.smithi144 on smithi144 2024-09-16T08:18:58.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:57 smithi144 ceph-mon[49149]: pgmap v38: 97 pgs: 97 active+clean; 333 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 7.7 KiB/s rd, 5.0 MiB/s wr, 503 op/s 2024-09-16T08:18:58.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:57 smithi144 ceph-mon[49149]: Upgrade: Updating crash.smithi144 (2/2) 2024-09-16T08:18:58.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:57 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:18:58.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:57 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "auth get-or-create", "entity": "client.crash.smithi144", "caps": ["mon", "profile crash", "mgr", "profile crash"]} : dispatch 2024-09-16T08:18:58.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:57 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-16T08:18:58.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:57 smithi144 ceph-mon[49149]: Deploying daemon crash.smithi144 on smithi144 2024-09-16T08:18:59.038 INFO:teuthology.orchestra.run.smithi028.stdout: 5 162848 10.67 MB/sec execute 237 sec latency 97.075 ms 2024-09-16T08:19:00.038 INFO:teuthology.orchestra.run.smithi028.stdout: 5 163119 10.67 MB/sec execute 238 sec latency 67.480 ms 2024-09-16T08:19:00.197 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:18:59 smithi028 ceph-mon[64115]: pgmap v39: 97 pgs: 97 active+clean; 337 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 7.2 MiB/s wr, 724 op/s 2024-09-16T08:19:00.249 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:18:59 smithi144 ceph-mon[49149]: pgmap v39: 97 pgs: 97 active+clean; 337 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 7.2 MiB/s wr, 724 op/s 2024-09-16T08:19:01.038 INFO:teuthology.orchestra.run.smithi028.stdout: 5 163379 10.66 MB/sec execute 239 sec latency 135.497 ms 2024-09-16T08:19:02.038 INFO:teuthology.orchestra.run.smithi028.stdout: 5 163698 10.66 MB/sec execute 240 sec latency 55.592 ms 2024-09-16T08:19:02.045 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:19:01 smithi028 ceph-mon[64115]: pgmap v40: 97 pgs: 97 active+clean; 337 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 7.1 KiB/s rd, 5.3 MiB/s wr, 518 op/s 2024-09-16T08:19:02.046 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:19:01 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:19:02.046 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:19:01 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:19:02.046 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:19:01 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-16T08:19:02.081 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:19:01 smithi144 ceph-mon[49149]: pgmap v40: 97 pgs: 97 active+clean; 337 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 7.1 KiB/s rd, 5.3 MiB/s wr, 518 op/s 2024-09-16T08:19:02.082 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:19:01 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:19:02.082 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:19:01 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:19:02.082 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:19:01 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-16T08:19:02.198 INFO:teuthology.orchestra.run.smithi028.stdout:true 2024-09-16T08:19:03.039 INFO:teuthology.orchestra.run.smithi028.stdout: 5 164061 10.64 MB/sec execute 241 sec latency 86.084 ms 2024-09-16T08:19:03.061 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:19:02 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-16T08:19:03.081 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:19:02 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-16T08:19:03.361 INFO:teuthology.orchestra.run.smithi028.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-16T08:19:03.361 INFO:teuthology.orchestra.run.smithi028.stdout:alertmanager.smithi028 smithi028 *:9093,9094 running (3m) 67s ago 12m 19.7M - 0.27.0 11f11916f8cd a14fcb4cb2b6 2024-09-16T08:19:03.361 INFO:teuthology.orchestra.run.smithi028.stdout:crash.smithi028 smithi028 starting - - - - 2024-09-16T08:19:03.361 INFO:teuthology.orchestra.run.smithi028.stdout:crash.smithi144 smithi144 starting - - - - 2024-09-16T08:19:03.361 INFO:teuthology.orchestra.run.smithi028.stdout:grafana.smithi028 smithi028 *:3000 running (3m) 67s ago 12m 43.8M - dad864ee21e9 b55ba8b944a1 2024-09-16T08:19:03.362 INFO:teuthology.orchestra.run.smithi028.stdout:mds.foofs.smithi028.srtgev smithi028 running (6m) 67s ago 6m 37.1M - 17.2.0 e1d6a67b021e 250e7a7f754c 2024-09-16T08:19:03.362 INFO:teuthology.orchestra.run.smithi028.stdout:mds.foofs.smithi144.oozgsi smithi144 running (6m) 54s ago 6m 16.5M - 17.2.0 e1d6a67b021e ecf4c2bb0fdc 2024-09-16T08:19:03.362 INFO:teuthology.orchestra.run.smithi028.stdout:mgr.smithi028.azvyzy smithi028 *:8443,9283,8765 running (2m) 67s ago 13m 488M - 19.3.0-4917-g8293d73f 425e316ba8eb e01d4382d03f 2024-09-16T08:19:03.362 INFO:teuthology.orchestra.run.smithi028.stdout:mgr.smithi144.utzszj smithi144 *:8443,9283,8765 running (101s) 54s ago 11m 452M - 19.3.0-4917-g8293d73f 425e316ba8eb 71f7f802bffb 2024-09-16T08:19:03.362 INFO:teuthology.orchestra.run.smithi028.stdout:mon.smithi028 smithi028 running (84s) 67s ago 13m 31.9M 2048M 19.3.0-4917-g8293d73f 425e316ba8eb 14efcedc0088 2024-09-16T08:19:03.362 INFO:teuthology.orchestra.run.smithi028.stdout:mon.smithi144 smithi144 running (59s) 54s ago 11m 26.2M 2048M 19.3.0-4917-g8293d73f 425e316ba8eb c081c5ed44b5 2024-09-16T08:19:03.362 INFO:teuthology.orchestra.run.smithi028.stdout:nfs.foo.0.0.smithi028.dkttgj smithi028 *:2049 running (6m) 67s ago 6m 274M - 4.0 e1d6a67b021e b5dccc45a6ed 2024-09-16T08:19:03.362 INFO:teuthology.orchestra.run.smithi028.stdout:nfs.foo.1.0.smithi144.epipkg smithi144 *:2049 running (6m) 54s ago 6m 86.9M - 4.0 e1d6a67b021e faa89e61f50e 2024-09-16T08:19:03.362 INFO:teuthology.orchestra.run.smithi028.stdout:node-exporter.smithi028 smithi028 *:9100 running (3m) 67s ago 12m 9977k - 1.7.0 72c9c2088986 9e2b8e8a63a9 2024-09-16T08:19:03.362 INFO:teuthology.orchestra.run.smithi028.stdout:node-exporter.smithi144 smithi144 *:9100 running (2m) 54s ago 11m 11.1M - 1.7.0 72c9c2088986 78834c5e992f 2024-09-16T08:19:03.362 INFO:teuthology.orchestra.run.smithi028.stdout:osd.0 smithi028 running (10m) 67s ago 10m 485M 1937M 17.2.0 e1d6a67b021e 8184fbc3871f 2024-09-16T08:19:03.362 INFO:teuthology.orchestra.run.smithi028.stdout:osd.1 smithi028 running (10m) 67s ago 10m 408M 1937M 17.2.0 e1d6a67b021e 6911d5c68d54 2024-09-16T08:19:03.363 INFO:teuthology.orchestra.run.smithi028.stdout:osd.2 smithi028 running (10m) 67s ago 10m 299M 1937M 17.2.0 e1d6a67b021e 90b27d8375a2 2024-09-16T08:19:03.363 INFO:teuthology.orchestra.run.smithi028.stdout:osd.3 smithi028 running (9m) 67s ago 9m 448M 1937M 17.2.0 e1d6a67b021e dac477d8b04f 2024-09-16T08:19:03.363 INFO:teuthology.orchestra.run.smithi028.stdout:osd.4 smithi144 running (9m) 54s ago 9m 477M 2704M 17.2.0 e1d6a67b021e 86719c60975c 2024-09-16T08:19:03.363 INFO:teuthology.orchestra.run.smithi028.stdout:osd.5 smithi144 running (8m) 54s ago 8m 536M 2704M 17.2.0 e1d6a67b021e 70452c145c60 2024-09-16T08:19:03.363 INFO:teuthology.orchestra.run.smithi028.stdout:osd.6 smithi144 running (8m) 54s ago 8m 455M 2704M 17.2.0 e1d6a67b021e 7f9981be792f 2024-09-16T08:19:03.363 INFO:teuthology.orchestra.run.smithi028.stdout:osd.7 smithi144 running (8m) 54s ago 8m 350M 2704M 17.2.0 e1d6a67b021e dada74ddbe4c 2024-09-16T08:19:03.363 INFO:teuthology.orchestra.run.smithi028.stdout:prometheus.smithi028 smithi028 *:9095 running (108s) 67s ago 11m 49.5M - 2.51.0 1d3b7f56885b 42667809df2b 2024-09-16T08:19:04.038 INFO:teuthology.orchestra.run.smithi028.stdout: 5 164333 10.62 MB/sec execute 242 sec latency 52.531 ms 2024-09-16T08:19:04.068 INFO:teuthology.orchestra.run.smithi028.stdout:{ 2024-09-16T08:19:04.068 INFO:teuthology.orchestra.run.smithi028.stdout: "mon": { 2024-09-16T08:19:04.068 INFO:teuthology.orchestra.run.smithi028.stdout: "ceph version 19.3.0-4917-g8293d73f (8293d73f8690540e843a81caec373f9cc29cf705) squid (dev)": 2 2024-09-16T08:19:04.068 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-09-16T08:19:04.068 INFO:teuthology.orchestra.run.smithi028.stdout: "mgr": { 2024-09-16T08:19:04.068 INFO:teuthology.orchestra.run.smithi028.stdout: "ceph version 19.3.0-4917-g8293d73f (8293d73f8690540e843a81caec373f9cc29cf705) squid (dev)": 2 2024-09-16T08:19:04.069 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-09-16T08:19:04.069 INFO:teuthology.orchestra.run.smithi028.stdout: "osd": { 2024-09-16T08:19:04.069 INFO:teuthology.orchestra.run.smithi028.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-09-16T08:19:04.069 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-09-16T08:19:04.069 INFO:teuthology.orchestra.run.smithi028.stdout: "mds": { 2024-09-16T08:19:04.069 INFO:teuthology.orchestra.run.smithi028.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-16T08:19:04.069 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-09-16T08:19:04.069 INFO:teuthology.orchestra.run.smithi028.stdout: "overall": { 2024-09-16T08:19:04.069 INFO:teuthology.orchestra.run.smithi028.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 10, 2024-09-16T08:19:04.069 INFO:teuthology.orchestra.run.smithi028.stdout: "ceph version 19.3.0-4917-g8293d73f (8293d73f8690540e843a81caec373f9cc29cf705) squid (dev)": 4 2024-09-16T08:19:04.069 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-09-16T08:19:04.069 INFO:teuthology.orchestra.run.smithi028.stdout:} 2024-09-16T08:19:04.078 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:19:03 smithi028 ceph-mon[64115]: from='client.34180 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T08:19:04.078 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:19:03 smithi028 ceph-mon[64115]: from='client.44129 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T08:19:04.078 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:19:03 smithi028 ceph-mon[64115]: pgmap v41: 97 pgs: 97 active+clean; 340 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 9.6 KiB/s rd, 6.0 MiB/s wr, 602 op/s 2024-09-16T08:19:04.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:19:03 smithi144 ceph-mon[49149]: from='client.34180 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T08:19:04.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:19:03 smithi144 ceph-mon[49149]: from='client.44129 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T08:19:04.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:19:03 smithi144 ceph-mon[49149]: pgmap v41: 97 pgs: 97 active+clean; 340 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 9.6 KiB/s rd, 6.0 MiB/s wr, 602 op/s 2024-09-16T08:19:04.584 INFO:teuthology.orchestra.run.smithi028.stdout:{ 2024-09-16T08:19:04.584 INFO:teuthology.orchestra.run.smithi028.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:f7ad55f9ccd17ba66aadb19b4d291e67b60ea24e5d8a0c0e92941fd59c81e4ce", 2024-09-16T08:19:04.584 INFO:teuthology.orchestra.run.smithi028.stdout: "in_progress": true, 2024-09-16T08:19:04.584 INFO:teuthology.orchestra.run.smithi028.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-16T08:19:04.584 INFO:teuthology.orchestra.run.smithi028.stdout: "services_complete": [ 2024-09-16T08:19:04.584 INFO:teuthology.orchestra.run.smithi028.stdout: "mon", 2024-09-16T08:19:04.584 INFO:teuthology.orchestra.run.smithi028.stdout: "mgr" 2024-09-16T08:19:04.584 INFO:teuthology.orchestra.run.smithi028.stdout: ], 2024-09-16T08:19:04.584 INFO:teuthology.orchestra.run.smithi028.stdout: "progress": "4/23 daemons upgraded", 2024-09-16T08:19:04.584 INFO:teuthology.orchestra.run.smithi028.stdout: "message": "Currently upgrading crash daemons", 2024-09-16T08:19:04.584 INFO:teuthology.orchestra.run.smithi028.stdout: "is_paused": false 2024-09-16T08:19:04.585 INFO:teuthology.orchestra.run.smithi028.stdout:} 2024-09-16T08:19:05.039 INFO:teuthology.orchestra.run.smithi028.stdout: 5 164732 10.63 MB/sec execute 243 sec latency 42.805 ms 2024-09-16T08:19:05.084 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:19:04 smithi028 ceph-mon[64115]: from='client.34188 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T08:19:05.084 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:19:04 smithi028 ceph-mon[64115]: from='client.? 172.21.15.28:0/2869549473' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-16T08:19:05.084 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:19:04 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:19:05.084 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:19:04 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:19:05.258 INFO:teuthology.orchestra.run.smithi028.stdout:HEALTH_OK 2024-09-16T08:19:05.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:19:04 smithi144 ceph-mon[49149]: from='client.34188 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T08:19:05.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:19:04 smithi144 ceph-mon[49149]: from='client.? 172.21.15.28:0/2869549473' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-16T08:19:05.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:19:04 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:19:05.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:19:04 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:19:06.041 INFO:teuthology.orchestra.run.smithi028.stdout: 5 165168 10.62 MB/sec execute 244 sec latency 61.296 ms 2024-09-16T08:19:06.098 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:19:05 smithi028 ceph-mon[64115]: from='client.34196 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T08:19:06.098 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:19:05 smithi028 ceph-mon[64115]: pgmap v42: 97 pgs: 97 active+clean; 342 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 5.2 KiB/s rd, 6.8 MiB/s wr, 679 op/s 2024-09-16T08:19:06.098 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:19:05 smithi028 ceph-mon[64115]: from='client.? 172.21.15.28:0/969779885' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-16T08:19:06.098 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:19:05 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:19:06.098 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:19:05 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:19:06.183 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:19:05 smithi144 ceph-mon[49149]: from='client.34196 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T08:19:06.183 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:19:05 smithi144 ceph-mon[49149]: pgmap v42: 97 pgs: 97 active+clean; 342 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 5.2 KiB/s rd, 6.8 MiB/s wr, 679 op/s 2024-09-16T08:19:06.183 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:19:05 smithi144 ceph-mon[49149]: from='client.? 172.21.15.28:0/969779885' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-16T08:19:06.183 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:19:05 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:19:06.183 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:19:05 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:19:07.040 INFO:teuthology.orchestra.run.smithi028.stdout: 5 165489 10.61 MB/sec execute 245 sec latency 87.578 ms 2024-09-16T08:19:08.041 INFO:teuthology.orchestra.run.smithi028.stdout: 5 165873 10.62 MB/sec execute 246 sec latency 118.745 ms 2024-09-16T08:19:08.162 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:19:07 smithi028 ceph-mon[64115]: pgmap v43: 97 pgs: 97 active+clean; 342 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 5.2 KiB/s rd, 4.2 MiB/s wr, 404 op/s 2024-09-16T08:19:08.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:19:07 smithi144 ceph-mon[49149]: pgmap v43: 97 pgs: 97 active+clean; 342 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 5.2 KiB/s rd, 4.2 MiB/s wr, 404 op/s 2024-09-16T08:19:09.041 INFO:teuthology.orchestra.run.smithi028.stdout: 5 166083 10.59 MB/sec execute 247 sec latency 123.519 ms 2024-09-16T08:19:09.959 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:19:09 smithi028 ceph-mon[64115]: pgmap v44: 97 pgs: 97 active+clean; 330 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 9.2 KiB/s rd, 6.2 MiB/s wr, 630 op/s 2024-09-16T08:19:09.959 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:19:09 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:19:09.959 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:19:09 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:19:09.959 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:19:09 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:19:10.041 INFO:teuthology.orchestra.run.smithi028.stdout: 5 166505 10.61 MB/sec execute 248 sec latency 47.141 ms 2024-09-16T08:19:10.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:19:09 smithi144 ceph-mon[49149]: pgmap v44: 97 pgs: 97 active+clean; 330 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 9.2 KiB/s rd, 6.2 MiB/s wr, 630 op/s 2024-09-16T08:19:10.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:19:09 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:19:10.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:19:09 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:19:10.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:19:09 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:19:11.019 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:19:10 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:19:11.019 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:19:10 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:19:11.019 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:19:10 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:19:11.041 INFO:teuthology.orchestra.run.smithi028.stdout: 5 166756 10.60 MB/sec execute 249 sec latency 51.341 ms 2024-09-16T08:19:11.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:19:10 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:19:11.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:19:10 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:19:11.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:19:10 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:19:12.041 INFO:teuthology.orchestra.run.smithi028.stdout: 5 167016 10.60 MB/sec execute 250 sec latency 50.704 ms 2024-09-16T08:19:12.045 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:19:11 smithi028 ceph-mon[64115]: pgmap v45: 97 pgs: 97 active+clean; 330 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 6.5 KiB/s rd, 4.0 MiB/s wr, 408 op/s 2024-09-16T08:19:12.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:19:11 smithi144 ceph-mon[49149]: pgmap v45: 97 pgs: 97 active+clean; 330 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 6.5 KiB/s rd, 4.0 MiB/s wr, 408 op/s 2024-09-16T08:19:13.041 INFO:teuthology.orchestra.run.smithi028.stdout: 5 167250 10.58 MB/sec execute 251 sec latency 54.620 ms 2024-09-16T08:19:14.041 INFO:teuthology.orchestra.run.smithi028.stdout: 5 167574 10.55 MB/sec execute 252 sec latency 60.436 ms 2024-09-16T08:19:14.081 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:19:13 smithi144 ceph-mon[49149]: pgmap v46: 97 pgs: 97 active+clean; 335 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 8.7 KiB/s rd, 4.8 MiB/s wr, 492 op/s 2024-09-16T08:19:14.124 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:19:13 smithi028 ceph-mon[64115]: pgmap v46: 97 pgs: 97 active+clean; 335 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 8.7 KiB/s rd, 4.8 MiB/s wr, 492 op/s 2024-09-16T08:19:15.041 INFO:teuthology.orchestra.run.smithi028.stdout: 5 167748 10.53 MB/sec execute 253 sec latency 77.218 ms 2024-09-16T08:19:16.018 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:19:15 smithi028 ceph-mon[64115]: pgmap v47: 97 pgs: 97 active+clean; 341 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 5.3 MiB/s wr, 500 op/s 2024-09-16T08:19:16.041 INFO:teuthology.orchestra.run.smithi028.stdout: 5 167976 10.50 MB/sec execute 254 sec latency 105.083 ms 2024-09-16T08:19:16.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:19:15 smithi144 ceph-mon[49149]: pgmap v47: 97 pgs: 97 active+clean; 341 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 5.3 MiB/s wr, 500 op/s 2024-09-16T08:19:17.041 INFO:teuthology.orchestra.run.smithi028.stdout: 5 168267 10.50 MB/sec execute 255 sec latency 109.096 ms 2024-09-16T08:19:17.294 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:19:17 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:19:17.294 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:19:17 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:19:17.294 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:19:17 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-16T08:19:17.294 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:19:17 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-16T08:19:17.294 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:19:17 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:19:17.294 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:19:17 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:19:17.294 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:19:17 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-16T08:19:17.295 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:19:17 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "versions"} : dispatch 2024-09-16T08:19:17.295 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:19:17 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "versions"} : dispatch 2024-09-16T08:19:17.295 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:19:17 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "versions"} : dispatch 2024-09-16T08:19:17.295 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:19:17 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:19:17.295 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:19:17 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi028"} : dispatch 2024-09-16T08:19:17.295 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:19:17 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi028"}]': finished 2024-09-16T08:19:17.295 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:19:17 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi144"} : dispatch 2024-09-16T08:19:17.295 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:19:17 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi144"}]': finished 2024-09-16T08:19:17.295 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:19:17 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16} : dispatch 2024-09-16T08:19:17.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:19:17 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:19:17.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:19:17 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:19:17.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:19:17 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-16T08:19:17.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:19:17 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-16T08:19:17.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:19:17 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:19:17.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:19:17 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:19:17.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:19:17 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-16T08:19:17.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:19:17 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "versions"} : dispatch 2024-09-16T08:19:17.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:19:17 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "versions"} : dispatch 2024-09-16T08:19:17.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:19:17 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "versions"} : dispatch 2024-09-16T08:19:17.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:19:17 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:19:17.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:19:17 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi028"} : dispatch 2024-09-16T08:19:17.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:19:17 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi028"}]': finished 2024-09-16T08:19:17.583 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:19:17 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi144"} : dispatch 2024-09-16T08:19:17.583 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:19:17 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi144"}]': finished 2024-09-16T08:19:17.583 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:19:17 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16} : dispatch 2024-09-16T08:19:18.041 INFO:teuthology.orchestra.run.smithi028.stdout: 5 168583 10.48 MB/sec execute 256 sec latency 97.927 ms 2024-09-16T08:19:18.310 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:19:18 smithi028 ceph-mon[64115]: Upgrade: Setting container_image for all crash 2024-09-16T08:19:18.310 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:19:18 smithi028 ceph-mon[64115]: pgmap v48: 97 pgs: 97 active+clean; 341 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.1 MiB/s wr, 402 op/s 2024-09-16T08:19:18.310 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:19:18 smithi028 ceph-mon[64115]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-09-16T08:19:18.310 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:19:18 smithi028 ceph-mon[64115]: Upgrade: osd.0 is safe to restart 2024-09-16T08:19:18.310 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:19:18 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-16T08:19:18.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:19:18 smithi144 ceph-mon[49149]: Upgrade: Setting container_image for all crash 2024-09-16T08:19:18.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:19:18 smithi144 ceph-mon[49149]: pgmap v48: 97 pgs: 97 active+clean; 341 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.1 MiB/s wr, 402 op/s 2024-09-16T08:19:18.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:19:18 smithi144 ceph-mon[49149]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-09-16T08:19:18.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:19:18 smithi144 ceph-mon[49149]: Upgrade: osd.0 is safe to restart 2024-09-16T08:19:18.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:19:18 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-16T08:19:19.041 INFO:teuthology.orchestra.run.smithi028.stdout: 5 168901 10.48 MB/sec execute 257 sec latency 49.561 ms 2024-09-16T08:19:20.041 INFO:teuthology.orchestra.run.smithi028.stdout: 5 169102 10.45 MB/sec execute 258 sec latency 54.416 ms 2024-09-16T08:19:20.114 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:19:20 smithi028 ceph-mon[64115]: pgmap v49: 97 pgs: 97 active+clean; 338 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 9.5 KiB/s rd, 5.6 MiB/s wr, 566 op/s 2024-09-16T08:19:20.114 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:19:20 smithi028 ceph-mon[64115]: Upgrade: Updating osd.0 2024-09-16T08:19:20.114 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:19:20 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:19:20.114 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:19:20 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "auth get", "entity": "osd.0"} : dispatch 2024-09-16T08:19:20.114 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:19:20 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-16T08:19:20.114 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:19:20 smithi028 ceph-mon[64115]: Deploying daemon osd.0 on smithi028 2024-09-16T08:19:20.114 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:19:20 smithi028 ceph-mon[64115]: mgrmap e43: smithi028.azvyzy(active, since 92s), standbys: smithi144.utzszj 2024-09-16T08:19:20.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:19:20 smithi144 ceph-mon[49149]: pgmap v49: 97 pgs: 97 active+clean; 338 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 9.5 KiB/s rd, 5.6 MiB/s wr, 566 op/s 2024-09-16T08:19:20.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:19:20 smithi144 ceph-mon[49149]: Upgrade: Updating osd.0 2024-09-16T08:19:20.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:19:20 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:19:20.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:19:20 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "auth get", "entity": "osd.0"} : dispatch 2024-09-16T08:19:20.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:19:20 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-16T08:19:20.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:19:20 smithi144 ceph-mon[49149]: Deploying daemon osd.0 on smithi028 2024-09-16T08:19:20.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:19:20 smithi144 ceph-mon[49149]: mgrmap e43: smithi028.azvyzy(active, since 92s), standbys: smithi144.utzszj 2024-09-16T08:19:21.041 INFO:teuthology.orchestra.run.smithi028.stdout: 5 169455 10.46 MB/sec execute 259 sec latency 69.213 ms 2024-09-16T08:19:22.042 INFO:teuthology.orchestra.run.smithi028.stdout: 5 169643 10.43 MB/sec execute 260 sec latency 57.506 ms 2024-09-16T08:19:22.183 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:19:22 smithi028 ceph-mon[64115]: pgmap v50: 97 pgs: 97 active+clean; 338 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 5.5 KiB/s rd, 3.5 MiB/s wr, 341 op/s 2024-09-16T08:19:22.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:19:22 smithi144 ceph-mon[49149]: pgmap v50: 97 pgs: 97 active+clean; 338 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 5.5 KiB/s rd, 3.5 MiB/s wr, 341 op/s 2024-09-16T08:19:23.041 INFO:teuthology.orchestra.run.smithi028.stdout: 5 169950 10.42 MB/sec execute 261 sec latency 61.706 ms 2024-09-16T08:19:23.197 INFO:journalctl@ceph.osd.0.smithi028.stdout:Sep 16 08:19:23 smithi028 systemd[1]: Stopping Ceph osd.0 for 5622b2fe-7402-11ef-bceb-c7b262605968... 2024-09-16T08:19:23.457 INFO:journalctl@ceph.osd.0.smithi028.stdout:Sep 16 08:19:23 smithi028 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-0[34239]: 2024-09-16T08:19:23.363+0000 7f65e8021700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-16T08:19:23.457 INFO:journalctl@ceph.osd.0.smithi028.stdout:Sep 16 08:19:23 smithi028 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-0[34239]: 2024-09-16T08:19:23.363+0000 7f65e8021700 -1 osd.0 72 *** Got signal Terminated *** 2024-09-16T08:19:23.457 INFO:journalctl@ceph.osd.0.smithi028.stdout:Sep 16 08:19:23 smithi028 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-0[34239]: 2024-09-16T08:19:23.363+0000 7f65e8021700 -1 osd.0 72 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-16T08:19:24.041 INFO:teuthology.orchestra.run.smithi028.stdout: 5 170196 10.42 MB/sec execute 262 sec latency 40.544 ms 2024-09-16T08:19:24.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:19:24 smithi144 ceph-mon[49149]: pgmap v51: 97 pgs: 97 active+clean; 339 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 7.6 KiB/s rd, 4.0 MiB/s wr, 365 op/s 2024-09-16T08:19:24.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:19:24 smithi144 ceph-mon[49149]: osd.0 marked itself down and dead 2024-09-16T08:19:24.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:19:24 smithi028 ceph-mon[64115]: pgmap v51: 97 pgs: 97 active+clean; 339 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 7.6 KiB/s rd, 4.0 MiB/s wr, 365 op/s 2024-09-16T08:19:24.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:19:24 smithi028 ceph-mon[64115]: osd.0 marked itself down and dead 2024-09-16T08:19:24.947 INFO:journalctl@ceph.osd.0.smithi028.stdout:Sep 16 08:19:24 smithi028 podman[68422]: 2024-09-16 08:19:24.692197911 +0000 UTC m=+1.474689539 container died 8184fbc3871fc3523f53927518439ce76ac67c6c36d32ab5520ad54c1ca922bc (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-0, ceph=True, io.k8s.display-name=CentOS Stream 8, distribution-scope=public, build-date=2022-05-03T08:36:31.336870, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, maintainer=Guillaume Abrioux , GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, name=centos-stream, com.redhat.component=centos-stream-container, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vendor=Red Hat, Inc., com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vcs-type=git, release=754, architecture=x86_64, version=8, GIT_REPO=https://github.com/ceph/ceph-container.git, io.openshift.tags=base centos centos-stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_BRANCH=HEAD, io.buildah.version=1.19.8, io.openshift.expose-services=, RELEASE=HEAD, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_CLEAN=True, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., CEPH_POINT_RELEASE=-17.2.0) 2024-09-16T08:19:25.041 INFO:teuthology.orchestra.run.smithi028.stdout: 5 170196 10.38 MB/sec execute 263 sec latency 1019.697 ms 2024-09-16T08:19:25.169 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:19:25 smithi028 ceph-mon[64115]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-16T08:19:25.170 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:19:25 smithi028 ceph-mon[64115]: osdmap e73: 8 total, 7 up, 8 in 2024-09-16T08:19:25.170 INFO:journalctl@ceph.osd.0.smithi028.stdout:Sep 16 08:19:25 smithi028 podman[68422]: 2024-09-16 08:19:25.1145549 +0000 UTC m=+1.897046532 container cleanup 8184fbc3871fc3523f53927518439ce76ac67c6c36d32ab5520ad54c1ca922bc (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-0, CEPH_POINT_RELEASE=-17.2.0, distribution-scope=public, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, RELEASE=HEAD, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, com.redhat.component=centos-stream-container, io.buildah.version=1.19.8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., version=8, GIT_BRANCH=HEAD, ceph=True, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vendor=Red Hat, Inc., maintainer=Guillaume Abrioux , GIT_CLEAN=True, release=754, name=centos-stream, io.openshift.expose-services=, build-date=2022-05-03T08:36:31.336870, io.openshift.tags=base centos centos-stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, architecture=x86_64, vcs-type=git, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.k8s.display-name=CentOS Stream 8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_REPO=https://github.com/ceph/ceph-container.git) 2024-09-16T08:19:25.170 INFO:journalctl@ceph.osd.0.smithi028.stdout:Sep 16 08:19:25 smithi028 bash[68422]: ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-0 2024-09-16T08:19:25.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:19:25 smithi144 ceph-mon[49149]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-16T08:19:25.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:19:25 smithi144 ceph-mon[49149]: osdmap e73: 8 total, 7 up, 8 in 2024-09-16T08:19:25.476 INFO:journalctl@ceph.osd.0.smithi028.stdout:Sep 16 08:19:25 smithi028 podman[68437]: 2024-09-16 08:19:25.459458788 +0000 UTC m=+0.762529812 container remove 8184fbc3871fc3523f53927518439ce76ac67c6c36d32ab5520ad54c1ca922bc (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-0, vendor=Red Hat, Inc., GIT_BRANCH=HEAD, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, name=centos-stream, build-date=2022-05-03T08:36:31.336870, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, CEPH_POINT_RELEASE=-17.2.0, vcs-type=git, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.expose-services=, ceph=True, io.openshift.tags=base centos centos-stream, com.redhat.component=centos-stream-container, GIT_REPO=https://github.com/ceph/ceph-container.git, RELEASE=HEAD, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_CLEAN=True, architecture=x86_64, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.k8s.display-name=CentOS Stream 8, distribution-scope=public, io.buildah.version=1.19.8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, version=8, maintainer=Guillaume Abrioux , release=754) 2024-09-16T08:19:25.981 INFO:journalctl@ceph.osd.0.smithi028.stdout:Sep 16 08:19:25 smithi028 podman[68523]: 2024-09-16 08:19:25.874163622 +0000 UTC m=+0.023833813 image pull 425e316ba8ebf74f04aa3640f7deb4fa3f293a78d0c8d097cb679eaf86ddca6c quay.ceph.io/ceph-ci/ceph@sha256:f7ad55f9ccd17ba66aadb19b4d291e67b60ea24e5d8a0c0e92941fd59c81e4ce 2024-09-16T08:19:26.041 INFO:teuthology.orchestra.run.smithi028.stdout: 5 170514 10.39 MB/sec execute 264 sec latency 1067.449 ms 2024-09-16T08:19:26.236 INFO:journalctl@ceph.osd.0.smithi028.stdout:Sep 16 08:19:26 smithi028 podman[68523]: 2024-09-16 08:19:26.013194444 +0000 UTC m=+0.162864624 container create 170e4eae571b23ee72d63561c35f65ba34d4e70b553e7990b64aa8d2b3eba95b (image=quay.ceph.io/ceph-ci/ceph@sha256:f7ad55f9ccd17ba66aadb19b4d291e67b60ea24e5d8a0c0e92941fd59c81e4ce, name=ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-0-deactivate, ceph=True, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, RELEASE=wip-guits-main-2024-09-13-1248-8293d73, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True) 2024-09-16T08:19:26.237 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:19:26 smithi028 ceph-mon[64115]: pgmap v53: 97 pgs: 13 stale+active+clean, 84 active+clean; 341 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 6.4 KiB/s rd, 3.8 MiB/s wr, 396 op/s 2024-09-16T08:19:26.237 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:19:26 smithi028 ceph-mon[64115]: osdmap e74: 8 total, 7 up, 8 in 2024-09-16T08:19:26.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:19:26 smithi144 ceph-mon[49149]: pgmap v53: 97 pgs: 13 stale+active+clean, 84 active+clean; 341 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 6.4 KiB/s rd, 3.8 MiB/s wr, 396 op/s 2024-09-16T08:19:26.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:19:26 smithi144 ceph-mon[49149]: osdmap e74: 8 total, 7 up, 8 in 2024-09-16T08:19:26.487 INFO:journalctl@ceph.osd.0.smithi028.stdout:Sep 16 08:19:26 smithi028 podman[68523]: 2024-09-16 08:19:26.347763717 +0000 UTC m=+0.497433917 container init 170e4eae571b23ee72d63561c35f65ba34d4e70b553e7990b64aa8d2b3eba95b (image=quay.ceph.io/ceph-ci/ceph@sha256:f7ad55f9ccd17ba66aadb19b4d291e67b60ea24e5d8a0c0e92941fd59c81e4ce, name=ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-0-deactivate, ceph=True, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, GIT_CLEAN=True, RELEASE=wip-guits-main-2024-09-13-1248-8293d73, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-16T08:19:26.487 INFO:journalctl@ceph.osd.0.smithi028.stdout:Sep 16 08:19:26 smithi028 podman[68523]: 2024-09-16 08:19:26.35318191 +0000 UTC m=+0.502852092 container start 170e4eae571b23ee72d63561c35f65ba34d4e70b553e7990b64aa8d2b3eba95b (image=quay.ceph.io/ceph-ci/ceph@sha256:f7ad55f9ccd17ba66aadb19b4d291e67b60ea24e5d8a0c0e92941fd59c81e4ce, name=ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-0-deactivate, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , RELEASE=wip-guits-main-2024-09-13-1248-8293d73, ceph=True, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0) 2024-09-16T08:19:26.487 INFO:journalctl@ceph.osd.0.smithi028.stdout:Sep 16 08:19:26 smithi028 podman[68523]: 2024-09-16 08:19:26.456400178 +0000 UTC m=+0.606070377 container attach 170e4eae571b23ee72d63561c35f65ba34d4e70b553e7990b64aa8d2b3eba95b (image=quay.ceph.io/ceph-ci/ceph@sha256:f7ad55f9ccd17ba66aadb19b4d291e67b60ea24e5d8a0c0e92941fd59c81e4ce, name=ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-0-deactivate, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, ceph=True, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, GIT_CLEAN=True, org.label-schema.vendor=CentOS, RELEASE=wip-guits-main-2024-09-13-1248-8293d73, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=) 2024-09-16T08:19:26.750 INFO:journalctl@ceph.osd.0.smithi028.stdout:Sep 16 08:19:26 smithi028 podman[68523]: 2024-09-16 08:19:26.689458718 +0000 UTC m=+0.839128899 container died 170e4eae571b23ee72d63561c35f65ba34d4e70b553e7990b64aa8d2b3eba95b (image=quay.ceph.io/ceph-ci/ceph@sha256:f7ad55f9ccd17ba66aadb19b4d291e67b60ea24e5d8a0c0e92941fd59c81e4ce, name=ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-0-deactivate, GIT_CLEAN=True, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=wip-guits-main-2024-09-13-1248-8293d73, ceph=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, org.label-schema.schema-version=1.0) 2024-09-16T08:19:27.041 INFO:teuthology.orchestra.run.smithi028.stdout: 5 170799 10.38 MB/sec execute 265 sec latency 53.277 ms 2024-09-16T08:19:27.774 INFO:journalctl@ceph.osd.0.smithi028.stdout:Sep 16 08:19:27 smithi028 podman[68544]: 2024-09-16 08:19:27.673016353 +0000 UTC m=+0.968618960 container remove 170e4eae571b23ee72d63561c35f65ba34d4e70b553e7990b64aa8d2b3eba95b (image=quay.ceph.io/ceph-ci/ceph@sha256:f7ad55f9ccd17ba66aadb19b4d291e67b60ea24e5d8a0c0e92941fd59c81e4ce, name=ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-0-deactivate, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, ceph=True, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, RELEASE=wip-guits-main-2024-09-13-1248-8293d73) 2024-09-16T08:19:27.774 INFO:journalctl@ceph.osd.0.smithi028.stdout:Sep 16 08:19:27 smithi028 systemd[1]: ceph-5622b2fe-7402-11ef-bceb-c7b262605968@osd.0.service: Deactivated successfully. 2024-09-16T08:19:27.774 INFO:journalctl@ceph.osd.0.smithi028.stdout:Sep 16 08:19:27 smithi028 systemd[1]: Stopped Ceph osd.0 for 5622b2fe-7402-11ef-bceb-c7b262605968. 2024-09-16T08:19:27.774 INFO:journalctl@ceph.osd.0.smithi028.stdout:Sep 16 08:19:27 smithi028 systemd[1]: ceph-5622b2fe-7402-11ef-bceb-c7b262605968@osd.0.service: Consumed 2min 53.052s CPU time. 2024-09-16T08:19:28.041 INFO:teuthology.orchestra.run.smithi028.stdout: 5 171098 10.36 MB/sec execute 266 sec latency 53.750 ms 2024-09-16T08:19:28.284 INFO:journalctl@ceph.osd.0.smithi028.stdout:Sep 16 08:19:28 smithi028 systemd[1]: Starting Ceph osd.0 for 5622b2fe-7402-11ef-bceb-c7b262605968... 2024-09-16T08:19:28.284 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:19:28 smithi028 ceph-mon[64115]: pgmap v55: 97 pgs: 13 stale+active+clean, 84 active+clean; 341 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 3.1 KiB/s rd, 2.6 MiB/s wr, 248 op/s 2024-09-16T08:19:28.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:19:28 smithi144 ceph-mon[49149]: pgmap v55: 97 pgs: 13 stale+active+clean, 84 active+clean; 341 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 3.1 KiB/s rd, 2.6 MiB/s wr, 248 op/s 2024-09-16T08:19:28.535 INFO:journalctl@ceph.osd.0.smithi028.stdout:Sep 16 08:19:28 smithi028 podman[68637]: 2024-09-16 08:19:28.19889905 +0000 UTC m=+0.021166253 image pull 425e316ba8ebf74f04aa3640f7deb4fa3f293a78d0c8d097cb679eaf86ddca6c quay.ceph.io/ceph-ci/ceph@sha256:f7ad55f9ccd17ba66aadb19b4d291e67b60ea24e5d8a0c0e92941fd59c81e4ce 2024-09-16T08:19:28.535 INFO:journalctl@ceph.osd.0.smithi028.stdout:Sep 16 08:19:28 smithi028 podman[68637]: 2024-09-16 08:19:28.313472182 +0000 UTC m=+0.135739366 container create b830b7fa0b07aa3115c237f8724f00f088b1269b0e6cec53290d660967eb7023 (image=quay.ceph.io/ceph-ci/ceph@sha256:f7ad55f9ccd17ba66aadb19b4d291e67b60ea24e5d8a0c0e92941fd59c81e4ce, name=ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-0-activate, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, RELEASE=wip-guits-main-2024-09-13-1248-8293d73, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, org.label-schema.license=GPLv2) 2024-09-16T08:19:28.803 INFO:journalctl@ceph.osd.0.smithi028.stdout:Sep 16 08:19:28 smithi028 podman[68637]: 2024-09-16 08:19:28.636770504 +0000 UTC m=+0.459037713 container init b830b7fa0b07aa3115c237f8724f00f088b1269b0e6cec53290d660967eb7023 (image=quay.ceph.io/ceph-ci/ceph@sha256:f7ad55f9ccd17ba66aadb19b4d291e67b60ea24e5d8a0c0e92941fd59c81e4ce, name=ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-0-activate, CEPH_POINT_RELEASE=, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, RELEASE=wip-guits-main-2024-09-13-1248-8293d73, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, ceph=True, org.label-schema.license=GPLv2, GIT_CLEAN=True) 2024-09-16T08:19:28.803 INFO:journalctl@ceph.osd.0.smithi028.stdout:Sep 16 08:19:28 smithi028 podman[68637]: 2024-09-16 08:19:28.646683742 +0000 UTC m=+0.468950935 container start b830b7fa0b07aa3115c237f8724f00f088b1269b0e6cec53290d660967eb7023 (image=quay.ceph.io/ceph-ci/ceph@sha256:f7ad55f9ccd17ba66aadb19b4d291e67b60ea24e5d8a0c0e92941fd59c81e4ce, name=ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-0-activate, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_BRANCH=HEAD, RELEASE=wip-guits-main-2024-09-13-1248-8293d73, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_CLEAN=True, io.buildah.version=1.37.2, ceph=True) 2024-09-16T08:19:28.803 INFO:journalctl@ceph.osd.0.smithi028.stdout:Sep 16 08:19:28 smithi028 podman[68637]: 2024-09-16 08:19:28.7450239 +0000 UTC m=+0.567291104 container attach b830b7fa0b07aa3115c237f8724f00f088b1269b0e6cec53290d660967eb7023 (image=quay.ceph.io/ceph-ci/ceph@sha256:f7ad55f9ccd17ba66aadb19b4d291e67b60ea24e5d8a0c0e92941fd59c81e4ce, name=ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-0-activate, RELEASE=wip-guits-main-2024-09-13-1248-8293d73, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_CLEAN=True, org.label-schema.vendor=CentOS) 2024-09-16T08:19:29.041 INFO:teuthology.orchestra.run.smithi028.stdout: 5 171287 10.33 MB/sec execute 267 sec latency 55.197 ms 2024-09-16T08:19:29.058 INFO:journalctl@ceph.osd.0.smithi028.stdout:Sep 16 08:19:28 smithi028 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-0-activate[68657]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-16T08:19:29.058 INFO:journalctl@ceph.osd.0.smithi028.stdout:Sep 16 08:19:28 smithi028 bash[68637]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-16T08:19:29.058 INFO:journalctl@ceph.osd.0.smithi028.stdout:Sep 16 08:19:28 smithi028 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-0-activate[68657]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-16T08:19:29.058 INFO:journalctl@ceph.osd.0.smithi028.stdout:Sep 16 08:19:28 smithi028 bash[68637]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-16T08:19:29.309 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:19:29 smithi028 ceph-mon[64115]: Health check failed: Degraded data redundancy: 208/1689 objects degraded (12.315%), 29 pgs degraded (PG_DEGRADED) 2024-09-16T08:19:29.405 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:19:29 smithi144 ceph-mon[49149]: Health check failed: Degraded data redundancy: 208/1689 objects degraded (12.315%), 29 pgs degraded (PG_DEGRADED) 2024-09-16T08:19:30.042 INFO:teuthology.orchestra.run.smithi028.stdout: 5 171442 10.31 MB/sec execute 268 sec latency 65.288 ms 2024-09-16T08:19:30.334 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:19:30 smithi028 ceph-mon[64115]: pgmap v56: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 343 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 5.5 KiB/s rd, 4.8 MiB/s wr, 454 op/s; 208/1689 objects degraded (12.315%) 2024-09-16T08:19:30.560 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:19:30 smithi144 ceph-mon[49149]: pgmap v56: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 343 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 5.5 KiB/s rd, 4.8 MiB/s wr, 454 op/s; 208/1689 objects degraded (12.315%) 2024-09-16T08:19:31.041 INFO:teuthology.orchestra.run.smithi028.stdout: 5 171724 10.31 MB/sec execute 269 sec latency 59.296 ms 2024-09-16T08:19:31.903 INFO:journalctl@ceph.osd.0.smithi028.stdout:Sep 16 08:19:31 smithi028 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-0-activate[68657]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-09-16T08:19:31.903 INFO:journalctl@ceph.osd.0.smithi028.stdout:Sep 16 08:19:31 smithi028 bash[68637]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-09-16T08:19:31.903 INFO:journalctl@ceph.osd.0.smithi028.stdout:Sep 16 08:19:31 smithi028 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-0-activate[68657]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-0 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2024-09-16T08:19:31.903 INFO:journalctl@ceph.osd.0.smithi028.stdout:Sep 16 08:19:31 smithi028 bash[68637]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-0 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2024-09-16T08:19:32.048 INFO:teuthology.orchestra.run.smithi028.stdout: 5 172072 10.29 MB/sec execute 270 sec latency 65.032 ms 2024-09-16T08:19:32.154 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:19:32 smithi028 ceph-mon[64115]: pgmap v57: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 343 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 2.4 KiB/s rd, 4.1 MiB/s wr, 418 op/s; 208/1689 objects degraded (12.315%) 2024-09-16T08:19:32.406 INFO:journalctl@ceph.osd.0.smithi028.stdout:Sep 16 08:19:32 smithi028 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-0-activate[68657]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2024-09-16T08:19:32.406 INFO:journalctl@ceph.osd.0.smithi028.stdout:Sep 16 08:19:32 smithi028 bash[68637]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2024-09-16T08:19:32.406 INFO:journalctl@ceph.osd.0.smithi028.stdout:Sep 16 08:19:32 smithi028 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-0-activate[68657]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-09-16T08:19:32.406 INFO:journalctl@ceph.osd.0.smithi028.stdout:Sep 16 08:19:32 smithi028 bash[68637]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-09-16T08:19:32.407 INFO:journalctl@ceph.osd.0.smithi028.stdout:Sep 16 08:19:32 smithi028 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-0-activate[68657]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-0/block 2024-09-16T08:19:32.407 INFO:journalctl@ceph.osd.0.smithi028.stdout:Sep 16 08:19:32 smithi028 bash[68637]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-0/block 2024-09-16T08:19:32.407 INFO:journalctl@ceph.osd.0.smithi028.stdout:Sep 16 08:19:32 smithi028 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-0-activate[68657]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-09-16T08:19:32.407 INFO:journalctl@ceph.osd.0.smithi028.stdout:Sep 16 08:19:32 smithi028 bash[68637]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-09-16T08:19:32.407 INFO:journalctl@ceph.osd.0.smithi028.stdout:Sep 16 08:19:32 smithi028 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-0-activate[68657]: --> ceph-volume raw activate successful for osd ID: 0 2024-09-16T08:19:32.407 INFO:journalctl@ceph.osd.0.smithi028.stdout:Sep 16 08:19:32 smithi028 bash[68637]: --> ceph-volume raw activate successful for osd ID: 0 2024-09-16T08:19:32.407 INFO:journalctl@ceph.osd.0.smithi028.stdout:Sep 16 08:19:32 smithi028 podman[68637]: 2024-09-16 08:19:32.233979457 +0000 UTC m=+4.056246645 container died b830b7fa0b07aa3115c237f8724f00f088b1269b0e6cec53290d660967eb7023 (image=quay.ceph.io/ceph-ci/ceph@sha256:f7ad55f9ccd17ba66aadb19b4d291e67b60ea24e5d8a0c0e92941fd59c81e4ce, name=ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-0-activate, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, ceph=True, org.label-schema.license=GPLv2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, RELEASE=wip-guits-main-2024-09-13-1248-8293d73, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=) 2024-09-16T08:19:32.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:19:32 smithi144 ceph-mon[49149]: pgmap v57: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 343 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 2.4 KiB/s rd, 4.1 MiB/s wr, 418 op/s; 208/1689 objects degraded (12.315%) 2024-09-16T08:19:33.049 INFO:teuthology.orchestra.run.smithi028.stdout: 5 172555 10.29 MB/sec execute 271 sec latency 53.269 ms 2024-09-16T08:19:33.431 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:19:33 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:19:33.431 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:19:33 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-16T08:19:33.431 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:19:33 smithi028 ceph-mon[64115]: pgmap v58: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 340 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 6.4 KiB/s rd, 2.5 MiB/s wr, 264 op/s; 203/1680 objects degraded (12.083%) 2024-09-16T08:19:33.431 INFO:journalctl@ceph.osd.0.smithi028.stdout:Sep 16 08:19:33 smithi028 podman[68836]: 2024-09-16 08:19:33.20432456 +0000 UTC m=+0.958024033 container remove b830b7fa0b07aa3115c237f8724f00f088b1269b0e6cec53290d660967eb7023 (image=quay.ceph.io/ceph-ci/ceph@sha256:f7ad55f9ccd17ba66aadb19b4d291e67b60ea24e5d8a0c0e92941fd59c81e4ce, name=ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-0-activate, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, RELEASE=wip-guits-main-2024-09-13-1248-8293d73, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, GIT_CLEAN=True, ceph=True, GIT_BRANCH=HEAD, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965) 2024-09-16T08:19:33.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:19:33 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:19:33.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:19:33 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-16T08:19:33.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:19:33 smithi144 ceph-mon[49149]: pgmap v58: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 340 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 6.4 KiB/s rd, 2.5 MiB/s wr, 264 op/s; 203/1680 objects degraded (12.083%) 2024-09-16T08:19:33.696 INFO:journalctl@ceph.osd.0.smithi028.stdout:Sep 16 08:19:33 smithi028 podman[68892]: 2024-09-16 08:19:33.378364826 +0000 UTC m=+0.028846664 image pull 425e316ba8ebf74f04aa3640f7deb4fa3f293a78d0c8d097cb679eaf86ddca6c quay.ceph.io/ceph-ci/ceph@sha256:f7ad55f9ccd17ba66aadb19b4d291e67b60ea24e5d8a0c0e92941fd59c81e4ce 2024-09-16T08:19:33.696 INFO:journalctl@ceph.osd.0.smithi028.stdout:Sep 16 08:19:33 smithi028 podman[68892]: 2024-09-16 08:19:33.630438336 +0000 UTC m=+0.280920165 container create 2fc02ba4f14f01fea22c133a927dc68cf30104c78c62455e439e3f26bb320214 (image=quay.ceph.io/ceph-ci/ceph@sha256:f7ad55f9ccd17ba66aadb19b4d291e67b60ea24e5d8a0c0e92941fd59c81e4ce, name=ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-0, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, RELEASE=wip-guits-main-2024-09-13-1248-8293d73, org.label-schema.license=GPLv2, GIT_CLEAN=True, io.buildah.version=1.37.2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux ) 2024-09-16T08:19:34.049 INFO:teuthology.orchestra.run.smithi028.stdout: 5 173025 10.31 MB/sec execute 272 sec latency 28.894 ms 2024-09-16T08:19:34.198 INFO:journalctl@ceph.osd.0.smithi028.stdout:Sep 16 08:19:34 smithi028 podman[68892]: 2024-09-16 08:19:34.018692307 +0000 UTC m=+0.669174137 container init 2fc02ba4f14f01fea22c133a927dc68cf30104c78c62455e439e3f26bb320214 (image=quay.ceph.io/ceph-ci/ceph@sha256:f7ad55f9ccd17ba66aadb19b4d291e67b60ea24e5d8a0c0e92941fd59c81e4ce, name=ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-0, org.label-schema.build-date=20240731, GIT_CLEAN=True, ceph=True, maintainer=Guillaume Abrioux , RELEASE=wip-guits-main-2024-09-13-1248-8293d73, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS) 2024-09-16T08:19:34.198 INFO:journalctl@ceph.osd.0.smithi028.stdout:Sep 16 08:19:34 smithi028 podman[68892]: 2024-09-16 08:19:34.024513946 +0000 UTC m=+0.674995770 container start 2fc02ba4f14f01fea22c133a927dc68cf30104c78c62455e439e3f26bb320214 (image=quay.ceph.io/ceph-ci/ceph@sha256:f7ad55f9ccd17ba66aadb19b4d291e67b60ea24e5d8a0c0e92941fd59c81e4ce, name=ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-0, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, CEPH_POINT_RELEASE=, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, RELEASE=wip-guits-main-2024-09-13-1248-8293d73, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux ) 2024-09-16T08:19:34.198 INFO:journalctl@ceph.osd.0.smithi028.stdout:Sep 16 08:19:34 smithi028 ceph-osd[68909]: -- 172.21.15.28:0/1145318979 <== mon.0 v2:172.21.15.28:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55a8176401e0 con 0x55a8167d5c00 2024-09-16T08:19:34.198 INFO:journalctl@ceph.osd.0.smithi028.stdout:Sep 16 08:19:34 smithi028 bash[68892]: 2fc02ba4f14f01fea22c133a927dc68cf30104c78c62455e439e3f26bb320214 2024-09-16T08:19:34.198 INFO:journalctl@ceph.osd.0.smithi028.stdout:Sep 16 08:19:34 smithi028 systemd[1]: Started Ceph osd.0 for 5622b2fe-7402-11ef-bceb-c7b262605968. 2024-09-16T08:19:35.049 INFO:teuthology.orchestra.run.smithi028.stdout: 5 173384 10.30 MB/sec execute 273 sec latency 39.434 ms 2024-09-16T08:19:35.472 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:19:35 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:19:35.473 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:19:35 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:19:35.473 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:19:35 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-16T08:19:35.473 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:19:35 smithi028 ceph-mon[64115]: pgmap v59: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 340 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 5.6 KiB/s rd, 3.8 MiB/s wr, 379 op/s; 200/1656 objects degraded (12.077%) 2024-09-16T08:19:35.473 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:19:35 smithi028 ceph-mon[64115]: Health check update: Degraded data redundancy: 203/1680 objects degraded (12.083%), 29 pgs degraded (PG_DEGRADED) 2024-09-16T08:19:35.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:19:35 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:19:35.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:19:35 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:19:35.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:19:35 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-16T08:19:35.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:19:35 smithi144 ceph-mon[49149]: pgmap v59: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 340 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 5.6 KiB/s rd, 3.8 MiB/s wr, 379 op/s; 200/1656 objects degraded (12.077%) 2024-09-16T08:19:35.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:19:35 smithi144 ceph-mon[49149]: Health check update: Degraded data redundancy: 203/1680 objects degraded (12.083%), 29 pgs degraded (PG_DEGRADED) 2024-09-16T08:19:35.848 INFO:teuthology.orchestra.run.smithi028.stdout:true 2024-09-16T08:19:36.049 INFO:teuthology.orchestra.run.smithi028.stdout: 5 173684 10.31 MB/sec execute 274 sec latency 54.074 ms 2024-09-16T08:19:36.225 INFO:journalctl@ceph.osd.0.smithi028.stdout:Sep 16 08:19:36 smithi028 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-0[68905]: 2024-09-16T08:19:36.166+0000 7f0ffef47640 -1 Falling back to public interface 2024-09-16T08:19:36.488 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:19:36 smithi028 ceph-mon[64115]: from='client.34206 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T08:19:36.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:19:36 smithi144 ceph-mon[49149]: from='client.34206 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T08:19:37.013 INFO:teuthology.orchestra.run.smithi028.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-16T08:19:37.013 INFO:teuthology.orchestra.run.smithi028.stdout:alertmanager.smithi028 smithi028 *:9093,9094 running (3m) 27s ago 12m 18.2M - 0.27.0 11f11916f8cd a14fcb4cb2b6 2024-09-16T08:19:37.013 INFO:teuthology.orchestra.run.smithi028.stdout:crash.smithi028 smithi028 running (42s) 27s ago 12m 6899k - 19.3.0-4917-g8293d73f 425e316ba8eb 4bb89df2a867 2024-09-16T08:19:37.013 INFO:teuthology.orchestra.run.smithi028.stdout:crash.smithi144 smithi144 running (36s) 32s ago 12m 6895k - 19.3.0-4917-g8293d73f 425e316ba8eb 24a96875d7af 2024-09-16T08:19:37.013 INFO:teuthology.orchestra.run.smithi028.stdout:grafana.smithi028 smithi028 *:3000 running (3m) 27s ago 12m 43.9M - dad864ee21e9 b55ba8b944a1 2024-09-16T08:19:37.013 INFO:teuthology.orchestra.run.smithi028.stdout:mds.foofs.smithi028.srtgev smithi028 running (7m) 27s ago 7m 37.5M - 17.2.0 e1d6a67b021e 250e7a7f754c 2024-09-16T08:19:37.013 INFO:teuthology.orchestra.run.smithi028.stdout:mds.foofs.smithi144.oozgsi smithi144 running (7m) 32s ago 7m 16.6M - 17.2.0 e1d6a67b021e ecf4c2bb0fdc 2024-09-16T08:19:37.014 INFO:teuthology.orchestra.run.smithi028.stdout:mgr.smithi028.azvyzy smithi028 *:8443,9283,8765 running (3m) 27s ago 14m 509M - 19.3.0-4917-g8293d73f 425e316ba8eb e01d4382d03f 2024-09-16T08:19:37.014 INFO:teuthology.orchestra.run.smithi028.stdout:mgr.smithi144.utzszj smithi144 *:8443,9283,8765 running (2m) 32s ago 12m 453M - 19.3.0-4917-g8293d73f 425e316ba8eb 71f7f802bffb 2024-09-16T08:19:37.014 INFO:teuthology.orchestra.run.smithi028.stdout:mon.smithi028 smithi028 running (117s) 27s ago 14m 43.3M 2048M 19.3.0-4917-g8293d73f 425e316ba8eb 14efcedc0088 2024-09-16T08:19:37.014 INFO:teuthology.orchestra.run.smithi028.stdout:mon.smithi144 smithi144 running (93s) 32s ago 12m 38.1M 2048M 19.3.0-4917-g8293d73f 425e316ba8eb c081c5ed44b5 2024-09-16T08:19:37.014 INFO:teuthology.orchestra.run.smithi028.stdout:nfs.foo.0.0.smithi028.dkttgj smithi028 *:2049 running (6m) 27s ago 6m 281M - 4.0 e1d6a67b021e b5dccc45a6ed 2024-09-16T08:19:37.014 INFO:teuthology.orchestra.run.smithi028.stdout:nfs.foo.1.0.smithi144.epipkg smithi144 *:2049 running (6m) 32s ago 6m 87.0M - 4.0 e1d6a67b021e faa89e61f50e 2024-09-16T08:19:37.014 INFO:teuthology.orchestra.run.smithi028.stdout:node-exporter.smithi028 smithi028 *:9100 running (3m) 27s ago 12m 9.85M - 1.7.0 72c9c2088986 9e2b8e8a63a9 2024-09-16T08:19:37.014 INFO:teuthology.orchestra.run.smithi028.stdout:node-exporter.smithi144 smithi144 *:9100 running (3m) 32s ago 12m 10.1M - 1.7.0 72c9c2088986 78834c5e992f 2024-09-16T08:19:37.014 INFO:teuthology.orchestra.run.smithi028.stdout:osd.0 smithi028 starting - - - 1937M 2024-09-16T08:19:37.014 INFO:teuthology.orchestra.run.smithi028.stdout:osd.1 smithi028 running (11m) 27s ago 11m 486M 1937M 17.2.0 e1d6a67b021e 6911d5c68d54 2024-09-16T08:19:37.014 INFO:teuthology.orchestra.run.smithi028.stdout:osd.2 smithi028 running (10m) 27s ago 10m 387M 1937M 17.2.0 e1d6a67b021e 90b27d8375a2 2024-09-16T08:19:37.014 INFO:teuthology.orchestra.run.smithi028.stdout:osd.3 smithi028 running (10m) 27s ago 10m 517M 1937M 17.2.0 e1d6a67b021e dac477d8b04f 2024-09-16T08:19:37.014 INFO:teuthology.orchestra.run.smithi028.stdout:osd.4 smithi144 running (9m) 32s ago 9m 527M 2704M 17.2.0 e1d6a67b021e 86719c60975c 2024-09-16T08:19:37.015 INFO:teuthology.orchestra.run.smithi028.stdout:osd.5 smithi144 running (9m) 32s ago 9m 334M 2704M 17.2.0 e1d6a67b021e 70452c145c60 2024-09-16T08:19:37.015 INFO:teuthology.orchestra.run.smithi028.stdout:osd.6 smithi144 running (9m) 32s ago 9m 533M 2704M 17.2.0 e1d6a67b021e 7f9981be792f 2024-09-16T08:19:37.015 INFO:teuthology.orchestra.run.smithi028.stdout:osd.7 smithi144 running (8m) 32s ago 8m 429M 2704M 17.2.0 e1d6a67b021e dada74ddbe4c 2024-09-16T08:19:37.015 INFO:teuthology.orchestra.run.smithi028.stdout:prometheus.smithi028 smithi028 *:9095 running (2m) 27s ago 11m 50.5M - 2.51.0 1d3b7f56885b 42667809df2b 2024-09-16T08:19:37.049 INFO:teuthology.orchestra.run.smithi028.stdout: 5 173928 10.30 MB/sec execute 275 sec latency 68.962 ms 2024-09-16T08:19:37.502 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:19:37 smithi028 ceph-mon[64115]: from='client.34210 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T08:19:37.502 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:19:37 smithi028 ceph-mon[64115]: pgmap v60: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 340 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 4.8 KiB/s rd, 3.2 MiB/s wr, 323 op/s; 200/1656 objects degraded (12.077%) 2024-09-16T08:19:37.502 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:19:37 smithi028 ceph-mon[64115]: from='client.34214 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T08:19:37.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:19:37 smithi144 ceph-mon[49149]: from='client.34210 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T08:19:37.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:19:37 smithi144 ceph-mon[49149]: pgmap v60: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 340 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 4.8 KiB/s rd, 3.2 MiB/s wr, 323 op/s; 200/1656 objects degraded (12.077%) 2024-09-16T08:19:37.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:19:37 smithi144 ceph-mon[49149]: from='client.34214 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T08:19:37.689 INFO:teuthology.orchestra.run.smithi028.stdout:{ 2024-09-16T08:19:37.689 INFO:teuthology.orchestra.run.smithi028.stdout: "mon": { 2024-09-16T08:19:37.689 INFO:teuthology.orchestra.run.smithi028.stdout: "ceph version 19.3.0-4917-g8293d73f (8293d73f8690540e843a81caec373f9cc29cf705) squid (dev)": 2 2024-09-16T08:19:37.689 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-09-16T08:19:37.689 INFO:teuthology.orchestra.run.smithi028.stdout: "mgr": { 2024-09-16T08:19:37.689 INFO:teuthology.orchestra.run.smithi028.stdout: "ceph version 19.3.0-4917-g8293d73f (8293d73f8690540e843a81caec373f9cc29cf705) squid (dev)": 2 2024-09-16T08:19:37.690 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-09-16T08:19:37.690 INFO:teuthology.orchestra.run.smithi028.stdout: "osd": { 2024-09-16T08:19:37.690 INFO:teuthology.orchestra.run.smithi028.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 7 2024-09-16T08:19:37.690 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-09-16T08:19:37.690 INFO:teuthology.orchestra.run.smithi028.stdout: "mds": { 2024-09-16T08:19:37.690 INFO:teuthology.orchestra.run.smithi028.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-16T08:19:37.690 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-09-16T08:19:37.690 INFO:teuthology.orchestra.run.smithi028.stdout: "overall": { 2024-09-16T08:19:37.690 INFO:teuthology.orchestra.run.smithi028.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 9, 2024-09-16T08:19:37.690 INFO:teuthology.orchestra.run.smithi028.stdout: "ceph version 19.3.0-4917-g8293d73f (8293d73f8690540e843a81caec373f9cc29cf705) squid (dev)": 4 2024-09-16T08:19:37.690 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-09-16T08:19:37.690 INFO:teuthology.orchestra.run.smithi028.stdout:} 2024-09-16T08:19:38.049 INFO:teuthology.orchestra.run.smithi028.stdout: 5 174167 10.30 MB/sec execute 276 sec latency 48.761 ms 2024-09-16T08:19:38.261 INFO:teuthology.orchestra.run.smithi028.stdout:{ 2024-09-16T08:19:38.261 INFO:teuthology.orchestra.run.smithi028.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:f7ad55f9ccd17ba66aadb19b4d291e67b60ea24e5d8a0c0e92941fd59c81e4ce", 2024-09-16T08:19:38.261 INFO:teuthology.orchestra.run.smithi028.stdout: "in_progress": true, 2024-09-16T08:19:38.261 INFO:teuthology.orchestra.run.smithi028.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-16T08:19:38.261 INFO:teuthology.orchestra.run.smithi028.stdout: "services_complete": [ 2024-09-16T08:19:38.261 INFO:teuthology.orchestra.run.smithi028.stdout: "crash", 2024-09-16T08:19:38.261 INFO:teuthology.orchestra.run.smithi028.stdout: "mon", 2024-09-16T08:19:38.261 INFO:teuthology.orchestra.run.smithi028.stdout: "mgr" 2024-09-16T08:19:38.262 INFO:teuthology.orchestra.run.smithi028.stdout: ], 2024-09-16T08:19:38.262 INFO:teuthology.orchestra.run.smithi028.stdout: "progress": "6/23 daemons upgraded", 2024-09-16T08:19:38.262 INFO:teuthology.orchestra.run.smithi028.stdout: "message": "Currently upgrading osd daemons", 2024-09-16T08:19:38.262 INFO:teuthology.orchestra.run.smithi028.stdout: "is_paused": false 2024-09-16T08:19:38.262 INFO:teuthology.orchestra.run.smithi028.stdout:} 2024-09-16T08:19:38.517 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:19:38 smithi028 ceph-mon[64115]: from='client.? 172.21.15.28:0/2035035849' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-16T08:19:38.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:19:38 smithi144 ceph-mon[49149]: from='client.? 172.21.15.28:0/2035035849' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-16T08:19:39.030 INFO:teuthology.orchestra.run.smithi028.stdout:HEALTH_WARN 1 osds down; Degraded data redundancy: 200/1656 objects degraded (12.077%), 29 pgs degraded 2024-09-16T08:19:39.031 INFO:teuthology.orchestra.run.smithi028.stdout:[WRN] OSD_DOWN: 1 osds down 2024-09-16T08:19:39.031 INFO:teuthology.orchestra.run.smithi028.stdout: osd.0 (root=default,host=smithi028) is down 2024-09-16T08:19:39.031 INFO:teuthology.orchestra.run.smithi028.stdout:[WRN] PG_DEGRADED: Degraded data redundancy: 200/1656 objects degraded (12.077%), 29 pgs degraded 2024-09-16T08:19:39.031 INFO:teuthology.orchestra.run.smithi028.stdout: pg 1.0 is active+undersized+degraded, acting [7,6] 2024-09-16T08:19:39.031 INFO:teuthology.orchestra.run.smithi028.stdout: pg 2.0 is active+undersized+degraded, acting [7,1] 2024-09-16T08:19:39.031 INFO:teuthology.orchestra.run.smithi028.stdout: pg 2.1 is active+undersized+degraded, acting [2,3] 2024-09-16T08:19:39.031 INFO:teuthology.orchestra.run.smithi028.stdout: pg 2.4 is active+undersized+degraded, acting [1,7] 2024-09-16T08:19:39.031 INFO:teuthology.orchestra.run.smithi028.stdout: pg 2.5 is active+undersized+degraded, acting [7,4] 2024-09-16T08:19:39.031 INFO:teuthology.orchestra.run.smithi028.stdout: pg 2.8 is active+undersized+degraded, acting [7,1] 2024-09-16T08:19:39.031 INFO:teuthology.orchestra.run.smithi028.stdout: pg 2.c is active+undersized+degraded, acting [2,5] 2024-09-16T08:19:39.031 INFO:teuthology.orchestra.run.smithi028.stdout: pg 2.f is active+undersized+degraded, acting [4,7] 2024-09-16T08:19:39.032 INFO:teuthology.orchestra.run.smithi028.stdout: pg 2.10 is active+undersized+degraded, acting [2,1] 2024-09-16T08:19:39.032 INFO:teuthology.orchestra.run.smithi028.stdout: pg 2.13 is active+undersized+degraded, acting [4,3] 2024-09-16T08:19:39.032 INFO:teuthology.orchestra.run.smithi028.stdout: pg 2.15 is active+undersized+degraded, acting [1,5] 2024-09-16T08:19:39.032 INFO:teuthology.orchestra.run.smithi028.stdout: pg 2.19 is active+undersized+degraded, acting [3,6] 2024-09-16T08:19:39.032 INFO:teuthology.orchestra.run.smithi028.stdout: pg 2.1d is active+undersized+degraded, acting [7,6] 2024-09-16T08:19:39.032 INFO:teuthology.orchestra.run.smithi028.stdout: pg 2.1e is active+undersized+degraded, acting [3,5] 2024-09-16T08:19:39.032 INFO:teuthology.orchestra.run.smithi028.stdout: pg 2.1f is active+undersized+degraded, acting [7,4] 2024-09-16T08:19:39.032 INFO:teuthology.orchestra.run.smithi028.stdout: pg 3.1 is active+undersized+degraded, acting [4,3] 2024-09-16T08:19:39.032 INFO:teuthology.orchestra.run.smithi028.stdout: pg 3.3 is active+undersized+degraded, acting [4,6] 2024-09-16T08:19:39.032 INFO:teuthology.orchestra.run.smithi028.stdout: pg 3.6 is active+undersized+degraded, acting [1,4] 2024-09-16T08:19:39.032 INFO:teuthology.orchestra.run.smithi028.stdout: pg 3.7 is active+undersized+degraded, acting [3,7] 2024-09-16T08:19:39.032 INFO:teuthology.orchestra.run.smithi028.stdout: pg 3.b is active+undersized+degraded, acting [3,4] 2024-09-16T08:19:39.032 INFO:teuthology.orchestra.run.smithi028.stdout: pg 3.f is active+undersized+degraded, acting [7,4] 2024-09-16T08:19:39.033 INFO:teuthology.orchestra.run.smithi028.stdout: pg 3.10 is active+undersized+degraded, acting [6,5] 2024-09-16T08:19:39.033 INFO:teuthology.orchestra.run.smithi028.stdout: pg 3.12 is active+undersized+degraded, acting [7,3] 2024-09-16T08:19:39.033 INFO:teuthology.orchestra.run.smithi028.stdout: pg 3.17 is active+undersized+degraded, acting [5,3] 2024-09-16T08:19:39.033 INFO:teuthology.orchestra.run.smithi028.stdout: pg 3.18 is active+undersized+degraded, acting [3,1] 2024-09-16T08:19:39.033 INFO:teuthology.orchestra.run.smithi028.stdout: pg 3.1b is active+undersized+degraded, acting [4,7] 2024-09-16T08:19:39.033 INFO:teuthology.orchestra.run.smithi028.stdout: pg 3.1f is active+undersized+degraded, acting [5,2] 2024-09-16T08:19:39.033 INFO:teuthology.orchestra.run.smithi028.stdout: pg 4.0 is active+undersized+degraded, acting [3,7] 2024-09-16T08:19:39.033 INFO:teuthology.orchestra.run.smithi028.stdout: pg 4.19 is active+undersized+degraded, acting [3,2] 2024-09-16T08:19:39.049 INFO:teuthology.orchestra.run.smithi028.stdout: 5 174460 10.28 MB/sec execute 277 sec latency 52.580 ms 2024-09-16T08:19:39.553 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:19:39 smithi028 ceph-mon[64115]: from='client.34222 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T08:19:39.553 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:19:39 smithi028 ceph-mon[64115]: pgmap v61: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 341 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 7.6 KiB/s rd, 5.4 MiB/s wr, 517 op/s; 204/1620 objects degraded (12.593%) 2024-09-16T08:19:39.553 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:19:39 smithi028 ceph-mon[64115]: from='client.? 172.21.15.28:0/3977843823' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-16T08:19:39.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:19:39 smithi144 ceph-mon[49149]: from='client.34222 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T08:19:39.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:19:39 smithi144 ceph-mon[49149]: pgmap v61: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 341 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 7.6 KiB/s rd, 5.4 MiB/s wr, 517 op/s; 204/1620 objects degraded (12.593%) 2024-09-16T08:19:39.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:19:39 smithi144 ceph-mon[49149]: from='client.? 172.21.15.28:0/3977843823' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-16T08:19:40.049 INFO:teuthology.orchestra.run.smithi028.stdout: 5 174763 10.26 MB/sec execute 278 sec latency 47.482 ms 2024-09-16T08:19:40.332 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:19:40 smithi028 ceph-mon[64115]: Health check update: Degraded data redundancy: 204/1620 objects degraded (12.593%), 29 pgs degraded (PG_DEGRADED) 2024-09-16T08:19:40.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:19:40 smithi144 ceph-mon[49149]: Health check update: Degraded data redundancy: 204/1620 objects degraded (12.593%), 29 pgs degraded (PG_DEGRADED) 2024-09-16T08:19:41.049 INFO:teuthology.orchestra.run.smithi028.stdout: 5 174968 10.24 MB/sec execute 279 sec latency 65.332 ms 2024-09-16T08:19:41.346 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:19:41 smithi028 ceph-mon[64115]: pgmap v62: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 341 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 3.9 MiB/s wr, 380 op/s; 204/1620 objects degraded (12.593%) 2024-09-16T08:19:41.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:19:41 smithi144 ceph-mon[49149]: pgmap v62: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 341 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 3.9 MiB/s wr, 380 op/s; 204/1620 objects degraded (12.593%) 2024-09-16T08:19:42.049 INFO:teuthology.orchestra.run.smithi028.stdout: 5 175222 10.23 MB/sec execute 280 sec latency 64.472 ms 2024-09-16T08:19:42.654 INFO:journalctl@ceph.osd.0.smithi028.stdout:Sep 16 08:19:42 smithi028 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-0[68905]: 2024-09-16T08:19:42.645+0000 7f0ffef47640 -1 osd.0 0 read_superblock omap replica is missing. 2024-09-16T08:19:43.049 INFO:teuthology.orchestra.run.smithi028.stdout: 5 175655 10.23 MB/sec execute 281 sec latency 48.865 ms 2024-09-16T08:19:43.931 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:19:43 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:19:43.931 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:19:43 smithi028 ceph-mon[64115]: pgmap v63: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 346 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 9.0 KiB/s rd, 4.5 MiB/s wr, 472 op/s; 205/1620 objects degraded (12.654%) 2024-09-16T08:19:43.931 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:19:43 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:19:44.049 INFO:teuthology.orchestra.run.smithi028.stdout: 5 176043 10.23 MB/sec execute 282 sec latency 50.952 ms 2024-09-16T08:19:44.182 INFO:journalctl@ceph.osd.0.smithi028.stdout:Sep 16 08:19:44 smithi028 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-0[68905]: 2024-09-16T08:19:44.005+0000 7f0ffef47640 -1 osd.0 72 log_to_monitors true 2024-09-16T08:19:44.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:19:43 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:19:44.349 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:19:43 smithi144 ceph-mon[49149]: pgmap v63: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 346 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 9.0 KiB/s rd, 4.5 MiB/s wr, 472 op/s; 205/1620 objects degraded (12.654%) 2024-09-16T08:19:44.349 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:19:43 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:19:44.948 INFO:journalctl@ceph.osd.0.smithi028.stdout:Sep 16 08:19:44 smithi028 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-0[68905]: 2024-09-16T08:19:44.914+0000 7f0ff6cf1640 -1 osd.0 72 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-16T08:19:44.949 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:19:44 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:19:44.949 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:19:44 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:19:44.949 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:19:44 smithi028 ceph-mon[64115]: from='osd.0 [v2:172.21.15.28:6802/1737962094,v1:172.21.15.28:6803/1737962094]' entity='osd.0' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]} : dispatch 2024-09-16T08:19:45.049 INFO:teuthology.orchestra.run.smithi028.stdout: 5 176435 10.23 MB/sec execute 283 sec latency 40.856 ms 2024-09-16T08:19:45.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:19:44 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:19:45.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:19:44 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:19:45.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:19:44 smithi144 ceph-mon[49149]: from='osd.0 [v2:172.21.15.28:6802/1737962094,v1:172.21.15.28:6803/1737962094]' entity='osd.0' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]} : dispatch 2024-09-16T08:19:45.972 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:19:45 smithi028 ceph-mon[64115]: pgmap v64: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 346 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 5.9 KiB/s rd, 5.2 MiB/s wr, 507 op/s; 201/1662 objects degraded (12.094%) 2024-09-16T08:19:45.972 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:19:45 smithi028 ceph-mon[64115]: from='osd.0 [v2:172.21.15.28:6802/1737962094,v1:172.21.15.28:6803/1737962094]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-09-16T08:19:45.972 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:19:45 smithi028 ceph-mon[64115]: osdmap e75: 8 total, 7 up, 8 in 2024-09-16T08:19:45.972 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:19:45 smithi028 ceph-mon[64115]: from='osd.0 [v2:172.21.15.28:6802/1737962094,v1:172.21.15.28:6803/1737962094]' entity='osd.0' cmd={"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi028", "root=default"]} : dispatch 2024-09-16T08:19:45.972 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:19:45 smithi028 ceph-mon[64115]: Health check update: Degraded data redundancy: 201/1662 objects degraded (12.094%), 29 pgs degraded (PG_DEGRADED) 2024-09-16T08:19:46.049 INFO:teuthology.orchestra.run.smithi028.stdout: 5 176757 10.22 MB/sec execute 284 sec latency 181.158 ms 2024-09-16T08:19:46.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:19:45 smithi144 ceph-mon[49149]: pgmap v64: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 346 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 5.9 KiB/s rd, 5.2 MiB/s wr, 507 op/s; 201/1662 objects degraded (12.094%) 2024-09-16T08:19:46.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:19:45 smithi144 ceph-mon[49149]: from='osd.0 [v2:172.21.15.28:6802/1737962094,v1:172.21.15.28:6803/1737962094]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-09-16T08:19:46.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:19:45 smithi144 ceph-mon[49149]: osdmap e75: 8 total, 7 up, 8 in 2024-09-16T08:19:46.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:19:45 smithi144 ceph-mon[49149]: from='osd.0 [v2:172.21.15.28:6802/1737962094,v1:172.21.15.28:6803/1737962094]' entity='osd.0' cmd={"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi028", "root=default"]} : dispatch 2024-09-16T08:19:46.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:19:45 smithi144 ceph-mon[49149]: Health check update: Degraded data redundancy: 201/1662 objects degraded (12.094%), 29 pgs degraded (PG_DEGRADED) 2024-09-16T08:19:47.049 INFO:teuthology.orchestra.run.smithi028.stdout: 5 176799 10.19 MB/sec execute 285 sec latency 1149.996 ms 2024-09-16T08:19:47.196 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:19:46 smithi028 ceph-mon[64115]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-16T08:19:47.196 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:19:46 smithi028 ceph-mon[64115]: osd.0 [v2:172.21.15.28:6802/1737962094,v1:172.21.15.28:6803/1737962094] boot 2024-09-16T08:19:47.196 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:19:46 smithi028 ceph-mon[64115]: osdmap e76: 8 total, 8 up, 8 in 2024-09-16T08:19:47.196 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:19:46 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "osd metadata", "id": 0} : dispatch 2024-09-16T08:19:47.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:19:46 smithi144 ceph-mon[49149]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-16T08:19:47.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:19:46 smithi144 ceph-mon[49149]: osd.0 [v2:172.21.15.28:6802/1737962094,v1:172.21.15.28:6803/1737962094] boot 2024-09-16T08:19:47.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:19:46 smithi144 ceph-mon[49149]: osdmap e76: 8 total, 8 up, 8 in 2024-09-16T08:19:47.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:19:46 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "osd metadata", "id": 0} : dispatch 2024-09-16T08:19:48.050 INFO:teuthology.orchestra.run.smithi028.stdout: 5 176922 10.17 MB/sec execute 286 sec latency 2011.919 ms 2024-09-16T08:19:48.126 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:19:47 smithi028 ceph-mon[64115]: pgmap v67: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 346 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 4.5 KiB/s rd, 2.5 MiB/s wr, 274 op/s; 201/1662 objects degraded (12.094%) 2024-09-16T08:19:48.126 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:19:47 smithi028 ceph-mon[64115]: osdmap e77: 8 total, 8 up, 8 in 2024-09-16T08:19:48.126 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:19:47 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:19:48.126 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:19:47 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-16T08:19:48.126 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:19:47 smithi028 ceph-mon[64115]: osdmap e78: 8 total, 8 up, 8 in 2024-09-16T08:19:48.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:19:47 smithi144 ceph-mon[49149]: pgmap v67: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 346 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 4.5 KiB/s rd, 2.5 MiB/s wr, 274 op/s; 201/1662 objects degraded (12.094%) 2024-09-16T08:19:48.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:19:47 smithi144 ceph-mon[49149]: osdmap e77: 8 total, 8 up, 8 in 2024-09-16T08:19:48.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:19:47 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:19:48.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:19:47 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-16T08:19:48.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:19:47 smithi144 ceph-mon[49149]: osdmap e78: 8 total, 8 up, 8 in 2024-09-16T08:19:49.049 INFO:teuthology.orchestra.run.smithi028.stdout: 5 177404 10.19 MB/sec execute 287 sec latency 42.940 ms 2024-09-16T08:19:50.050 INFO:teuthology.orchestra.run.smithi028.stdout: 5 177657 10.18 MB/sec execute 288 sec latency 896.847 ms 2024-09-16T08:19:50.134 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:19:49 smithi028 ceph-mon[64115]: pgmap v70: 97 pgs: 1 active+recovering+degraded, 11 remapped+peering, 3 active+recovery_wait+degraded, 5 active+undersized, 3 active+undersized+degraded, 74 active+clean; 346 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 4.5 KiB/s rd, 4.5 MiB/s wr, 461 op/s; 79/1677 objects degraded (4.711%); 22 B/s, 5 objects/s recovering 2024-09-16T08:19:50.135 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:19:49 smithi028 ceph-mon[64115]: osdmap e79: 8 total, 8 up, 8 in 2024-09-16T08:19:50.135 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:19:49 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:19:50.135 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:19:49 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:19:50.135 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:19:49 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-16T08:19:50.135 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:19:49 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-16T08:19:50.135 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:19:49 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:19:50.136 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:19:49 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:19:50.136 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:19:49 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-16T08:19:50.136 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:19:49 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "versions"} : dispatch 2024-09-16T08:19:50.136 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:19:49 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "versions"} : dispatch 2024-09-16T08:19:50.136 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:19:49 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "versions"} : dispatch 2024-09-16T08:19:50.137 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:19:49 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16} : dispatch 2024-09-16T08:19:50.137 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:19:49 smithi028 ceph-mon[64115]: Health check update: Degraded data redundancy: 79/1677 objects degraded (4.711%), 7 pgs degraded (PG_DEGRADED) 2024-09-16T08:19:50.137 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:19:49 smithi028 ceph-mon[64115]: osdmap e80: 8 total, 8 up, 8 in 2024-09-16T08:19:50.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:19:49 smithi144 ceph-mon[49149]: pgmap v70: 97 pgs: 1 active+recovering+degraded, 11 remapped+peering, 3 active+recovery_wait+degraded, 5 active+undersized, 3 active+undersized+degraded, 74 active+clean; 346 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 4.5 KiB/s rd, 4.5 MiB/s wr, 461 op/s; 79/1677 objects degraded (4.711%); 22 B/s, 5 objects/s recovering 2024-09-16T08:19:50.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:19:49 smithi144 ceph-mon[49149]: osdmap e79: 8 total, 8 up, 8 in 2024-09-16T08:19:50.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:19:49 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:19:50.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:19:49 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:19:50.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:19:49 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-16T08:19:50.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:19:49 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-16T08:19:50.333 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:19:49 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:19:50.333 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:19:49 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:19:50.333 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:19:49 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-16T08:19:50.333 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:19:49 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "versions"} : dispatch 2024-09-16T08:19:50.333 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:19:49 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "versions"} : dispatch 2024-09-16T08:19:50.333 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:19:49 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "versions"} : dispatch 2024-09-16T08:19:50.333 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:19:49 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16} : dispatch 2024-09-16T08:19:50.333 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:19:49 smithi144 ceph-mon[49149]: Health check update: Degraded data redundancy: 79/1677 objects degraded (4.711%), 7 pgs degraded (PG_DEGRADED) 2024-09-16T08:19:50.333 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:19:49 smithi144 ceph-mon[49149]: osdmap e80: 8 total, 8 up, 8 in 2024-09-16T08:19:50.900 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:19:50 smithi028 ceph-mon[64115]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-09-16T08:19:50.900 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:19:50 smithi028 ceph-mon[64115]: Upgrade: unsafe to stop osd(s) at this time (6 PGs are or would become offline) 2024-09-16T08:19:50.900 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:19:50 smithi028 ceph-mon[64115]: osdmap e81: 8 total, 8 up, 8 in 2024-09-16T08:19:51.050 INFO:teuthology.orchestra.run.smithi028.stdout: 5 178088 10.19 MB/sec execute 289 sec latency 131.029 ms 2024-09-16T08:19:51.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:19:50 smithi144 ceph-mon[49149]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-09-16T08:19:51.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:19:50 smithi144 ceph-mon[49149]: Upgrade: unsafe to stop osd(s) at this time (6 PGs are or would become offline) 2024-09-16T08:19:51.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:19:50 smithi144 ceph-mon[49149]: osdmap e81: 8 total, 8 up, 8 in 2024-09-16T08:19:51.910 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:19:51 smithi028 ceph-mon[64115]: pgmap v73: 97 pgs: 3 peering, 1 active+recovering+degraded, 8 remapped+peering, 3 active+recovery_wait+degraded, 82 active+clean; 345 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 4.5 KiB/s rd, 4.6 MiB/s wr, 470 op/s; 56/1665 objects degraded (3.363%); 22 B/s, 23 objects/s recovering 2024-09-16T08:19:51.910 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:19:51 smithi028 ceph-mon[64115]: osdmap e82: 8 total, 8 up, 8 in 2024-09-16T08:19:52.050 INFO:teuthology.orchestra.run.smithi028.stdout: 5 178449 10.18 MB/sec execute 290 sec latency 925.883 ms 2024-09-16T08:19:52.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:19:51 smithi144 ceph-mon[49149]: pgmap v73: 97 pgs: 3 peering, 1 active+recovering+degraded, 8 remapped+peering, 3 active+recovery_wait+degraded, 82 active+clean; 345 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 4.5 KiB/s rd, 4.6 MiB/s wr, 470 op/s; 56/1665 objects degraded (3.363%); 22 B/s, 23 objects/s recovering 2024-09-16T08:19:52.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:19:51 smithi144 ceph-mon[49149]: osdmap e82: 8 total, 8 up, 8 in 2024-09-16T08:19:53.050 INFO:teuthology.orchestra.run.smithi028.stdout: 5 178926 10.20 MB/sec execute 291 sec latency 31.789 ms 2024-09-16T08:19:54.050 INFO:teuthology.orchestra.run.smithi028.stdout: 5 179399 10.20 MB/sec execute 292 sec latency 41.860 ms 2024-09-16T08:19:54.051 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:19:53 smithi028 ceph-mon[64115]: pgmap v76: 97 pgs: 3 peering, 1 active+recovering+degraded, 6 remapped+peering, 1 active+recovery_wait+degraded, 86 active+clean; 349 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 9.2 KiB/s rd, 2.2 MiB/s wr, 250 op/s; 22/1674 objects degraded (1.314%); 2.0 MiB/s, 27 objects/s recovering 2024-09-16T08:19:54.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:19:53 smithi144 ceph-mon[49149]: pgmap v76: 97 pgs: 3 peering, 1 active+recovering+degraded, 6 remapped+peering, 1 active+recovery_wait+degraded, 86 active+clean; 349 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 9.2 KiB/s rd, 2.2 MiB/s wr, 250 op/s; 22/1674 objects degraded (1.314%); 2.0 MiB/s, 27 objects/s recovering 2024-09-16T08:19:55.050 INFO:teuthology.orchestra.run.smithi028.stdout: 5 179916 10.20 MB/sec execute 293 sec latency 34.285 ms 2024-09-16T08:19:55.067 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:19:54 smithi028 ceph-mon[64115]: Health check update: Degraded data redundancy: 22/1674 objects degraded (1.314%), 2 pgs degraded (PG_DEGRADED) 2024-09-16T08:19:55.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:19:54 smithi144 ceph-mon[49149]: Health check update: Degraded data redundancy: 22/1674 objects degraded (1.314%), 2 pgs degraded (PG_DEGRADED) 2024-09-16T08:19:56.050 INFO:teuthology.orchestra.run.smithi028.stdout: 5 180348 10.22 MB/sec execute 294 sec latency 42.908 ms 2024-09-16T08:19:56.076 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:19:55 smithi028 ceph-mon[64115]: pgmap v77: 97 pgs: 3 peering, 2 remapped+peering, 92 active+clean; 346 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.5 MiB/s wr, 460 op/s; 2.5 MiB/s, 30 objects/s recovering 2024-09-16T08:19:56.076 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:19:55 smithi028 ceph-mon[64115]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 22/1674 objects degraded (1.314%), 2 pgs degraded) 2024-09-16T08:19:56.076 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:19:55 smithi028 ceph-mon[64115]: Cluster is now healthy 2024-09-16T08:19:56.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:19:55 smithi144 ceph-mon[49149]: pgmap v77: 97 pgs: 3 peering, 2 remapped+peering, 92 active+clean; 346 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.5 MiB/s wr, 460 op/s; 2.5 MiB/s, 30 objects/s recovering 2024-09-16T08:19:56.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:19:55 smithi144 ceph-mon[49149]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 22/1674 objects degraded (1.314%), 2 pgs degraded) 2024-09-16T08:19:56.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:19:55 smithi144 ceph-mon[49149]: Cluster is now healthy 2024-09-16T08:19:57.050 INFO:teuthology.orchestra.run.smithi028.stdout: 5 180830 10.23 MB/sec execute 295 sec latency 41.427 ms 2024-09-16T08:19:58.050 INFO:teuthology.orchestra.run.smithi028.stdout: 5 181186 10.23 MB/sec execute 296 sec latency 54.005 ms 2024-09-16T08:19:58.122 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:19:57 smithi028 ceph-mon[64115]: pgmap v78: 97 pgs: 97 active+clean; 347 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 5.3 KiB/s rd, 4.4 MiB/s wr, 419 op/s; 2.2 MiB/s, 26 objects/s recovering 2024-09-16T08:19:58.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:19:57 smithi144 ceph-mon[49149]: pgmap v78: 97 pgs: 97 active+clean; 347 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 5.3 KiB/s rd, 4.4 MiB/s wr, 419 op/s; 2.2 MiB/s, 26 objects/s recovering 2024-09-16T08:19:59.052 INFO:teuthology.orchestra.run.smithi028.stdout: 5 181726 10.23 MB/sec execute 297 sec latency 24.459 ms 2024-09-16T08:20:00.052 INFO:teuthology.orchestra.run.smithi028.stdout: 5 182377 10.27 MB/sec execute 298 sec latency 27.657 ms 2024-09-16T08:20:00.136 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:19:59 smithi028 ceph-mon[64115]: pgmap v79: 97 pgs: 97 active+clean; 348 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 7.4 MiB/s wr, 741 op/s; 1.9 MiB/s, 23 objects/s recovering 2024-09-16T08:20:00.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:19:59 smithi144 ceph-mon[49149]: pgmap v79: 97 pgs: 97 active+clean; 348 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 7.4 MiB/s wr, 741 op/s; 1.9 MiB/s, 23 objects/s recovering 2024-09-16T08:20:01.052 INFO:teuthology.orchestra.run.smithi028.stdout: 5 183152 10.28 MB/sec execute 299 sec latency 27.107 ms 2024-09-16T08:20:01.139 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:20:00 smithi028 ceph-mon[64115]: overall HEALTH_OK 2024-09-16T08:20:01.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:20:00 smithi144 ceph-mon[49149]: overall HEALTH_OK 2024-09-16T08:20:02.053 INFO:teuthology.orchestra.run.smithi028.stdout: 5 183547 10.28 MB/sec execute 300 sec latency 50.604 ms 2024-09-16T08:20:02.155 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:20:01 smithi028 ceph-mon[64115]: pgmap v80: 97 pgs: 97 active+clean; 350 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 8.8 KiB/s rd, 6.5 MiB/s wr, 622 op/s; 1.5 MiB/s, 18 objects/s recovering 2024-09-16T08:20:02.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:20:01 smithi144 ceph-mon[49149]: pgmap v80: 97 pgs: 97 active+clean; 350 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 8.8 KiB/s rd, 6.5 MiB/s wr, 622 op/s; 1.5 MiB/s, 18 objects/s recovering 2024-09-16T08:20:03.053 INFO:teuthology.orchestra.run.smithi028.stdout: 5 183822 10.28 MB/sec execute 301 sec latency 59.773 ms 2024-09-16T08:20:03.435 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:20:03 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:20:03.435 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:20:03 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-16T08:20:03.435 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:20:03 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:20:03.435 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:20:03 smithi028 ceph-mon[64115]: pgmap v81: 97 pgs: 97 active+clean; 350 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 13 KiB/s rd, 6.5 MiB/s wr, 596 op/s; 1.4 MiB/s, 16 objects/s recovering 2024-09-16T08:20:03.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:20:03 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:20:03.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:20:03 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-16T08:20:03.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:20:03 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:20:03.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:20:03 smithi144 ceph-mon[49149]: pgmap v81: 97 pgs: 97 active+clean; 350 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 13 KiB/s rd, 6.5 MiB/s wr, 596 op/s; 1.4 MiB/s, 16 objects/s recovering 2024-09-16T08:20:04.053 INFO:teuthology.orchestra.run.smithi028.stdout: 5 184175 10.28 MB/sec execute 302 sec latency 53.404 ms 2024-09-16T08:20:04.718 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:20:04 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16} : dispatch 2024-09-16T08:20:05.053 INFO:teuthology.orchestra.run.smithi028.stdout: 5 184484 10.28 MB/sec execute 303 sec latency 84.520 ms 2024-09-16T08:20:05.081 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:20:04 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16} : dispatch 2024-09-16T08:20:05.725 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:20:05 smithi028 ceph-mon[64115]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-09-16T08:20:05.725 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:20:05 smithi028 ceph-mon[64115]: Upgrade: osd.1 is safe to restart 2024-09-16T08:20:05.725 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:20:05 smithi028 ceph-mon[64115]: pgmap v82: 97 pgs: 97 active+clean; 347 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 8.9 KiB/s rd, 7.6 MiB/s wr, 743 op/s; 582 KiB/s, 12 objects/s recovering 2024-09-16T08:20:06.053 INFO:teuthology.orchestra.run.smithi028.stdout: 5 185040 10.30 MB/sec execute 304 sec latency 33.702 ms 2024-09-16T08:20:06.081 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:20:05 smithi144 ceph-mon[49149]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-09-16T08:20:06.081 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:20:05 smithi144 ceph-mon[49149]: Upgrade: osd.1 is safe to restart 2024-09-16T08:20:06.081 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:20:05 smithi144 ceph-mon[49149]: pgmap v82: 97 pgs: 97 active+clean; 347 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 8.9 KiB/s rd, 7.6 MiB/s wr, 743 op/s; 582 KiB/s, 12 objects/s recovering 2024-09-16T08:20:07.053 INFO:teuthology.orchestra.run.smithi028.stdout: 5 185633 10.32 MB/sec execute 305 sec latency 33.677 ms 2024-09-16T08:20:08.001 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:20:07 smithi028 ceph-mon[64115]: pgmap v83: 97 pgs: 97 active+clean; 345 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 8.9 KiB/s rd, 6.4 MiB/s wr, 619 op/s; 0 B/s, 6 objects/s recovering 2024-09-16T08:20:08.001 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:20:07 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:20:08.001 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:20:07 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "auth get", "entity": "osd.1"} : dispatch 2024-09-16T08:20:08.001 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:20:07 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-16T08:20:08.053 INFO:teuthology.orchestra.run.smithi028.stdout: 5 185900 10.31 MB/sec execute 306 sec latency 49.407 ms 2024-09-16T08:20:08.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:20:07 smithi144 ceph-mon[49149]: pgmap v83: 97 pgs: 97 active+clean; 345 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 8.9 KiB/s rd, 6.4 MiB/s wr, 619 op/s; 0 B/s, 6 objects/s recovering 2024-09-16T08:20:08.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:20:07 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:20:08.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:20:07 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "auth get", "entity": "osd.1"} : dispatch 2024-09-16T08:20:08.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:20:07 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-16T08:20:09.008 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:20:08 smithi028 ceph-mon[64115]: Upgrade: Updating osd.1 2024-09-16T08:20:09.008 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:20:08 smithi028 ceph-mon[64115]: Deploying daemon osd.1 on smithi028 2024-09-16T08:20:09.053 INFO:teuthology.orchestra.run.smithi028.stdout: 5 186178 10.30 MB/sec execute 307 sec latency 60.902 ms 2024-09-16T08:20:09.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:20:08 smithi144 ceph-mon[49149]: Upgrade: Updating osd.1 2024-09-16T08:20:09.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:20:08 smithi144 ceph-mon[49149]: Deploying daemon osd.1 on smithi028 2024-09-16T08:20:09.670 INFO:teuthology.orchestra.run.smithi028.stdout:true 2024-09-16T08:20:10.039 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:20:09 smithi028 ceph-mon[64115]: pgmap v84: 97 pgs: 97 active+clean; 351 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 12 KiB/s rd, 8.3 MiB/s wr, 825 op/s 2024-09-16T08:20:10.053 INFO:teuthology.orchestra.run.smithi028.stdout: 5 186452 10.29 MB/sec execute 308 sec latency 45.779 ms 2024-09-16T08:20:10.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:20:09 smithi144 ceph-mon[49149]: pgmap v84: 97 pgs: 97 active+clean; 351 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 12 KiB/s rd, 8.3 MiB/s wr, 825 op/s 2024-09-16T08:20:10.845 INFO:teuthology.orchestra.run.smithi028.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-16T08:20:10.845 INFO:teuthology.orchestra.run.smithi028.stdout:alertmanager.smithi028 smithi028 *:9093,9094 running (4m) 28s ago 13m 18.2M - 0.27.0 11f11916f8cd a14fcb4cb2b6 2024-09-16T08:20:10.845 INFO:teuthology.orchestra.run.smithi028.stdout:crash.smithi028 smithi028 running (76s) 28s ago 13m 6899k - 19.3.0-4917-g8293d73f 425e316ba8eb 4bb89df2a867 2024-09-16T08:20:10.845 INFO:teuthology.orchestra.run.smithi028.stdout:crash.smithi144 smithi144 running (70s) 66s ago 12m 6895k - 19.3.0-4917-g8293d73f 425e316ba8eb 24a96875d7af 2024-09-16T08:20:10.845 INFO:teuthology.orchestra.run.smithi028.stdout:grafana.smithi028 smithi028 *:3000 running (4m) 28s ago 13m 44.1M - dad864ee21e9 b55ba8b944a1 2024-09-16T08:20:10.845 INFO:teuthology.orchestra.run.smithi028.stdout:mds.foofs.smithi028.srtgev smithi028 running (7m) 28s ago 7m 37.4M - 17.2.0 e1d6a67b021e 250e7a7f754c 2024-09-16T08:20:10.845 INFO:teuthology.orchestra.run.smithi028.stdout:mds.foofs.smithi144.oozgsi smithi144 running (7m) 66s ago 7m 16.6M - 17.2.0 e1d6a67b021e ecf4c2bb0fdc 2024-09-16T08:20:10.845 INFO:teuthology.orchestra.run.smithi028.stdout:mgr.smithi028.azvyzy smithi028 *:8443,9283,8765 running (3m) 28s ago 14m 511M - 19.3.0-4917-g8293d73f 425e316ba8eb e01d4382d03f 2024-09-16T08:20:10.846 INFO:teuthology.orchestra.run.smithi028.stdout:mgr.smithi144.utzszj smithi144 *:8443,9283,8765 running (2m) 66s ago 12m 453M - 19.3.0-4917-g8293d73f 425e316ba8eb 71f7f802bffb 2024-09-16T08:20:10.846 INFO:teuthology.orchestra.run.smithi028.stdout:mon.smithi028 smithi028 running (2m) 28s ago 14m 46.1M 2048M 19.3.0-4917-g8293d73f 425e316ba8eb 14efcedc0088 2024-09-16T08:20:10.846 INFO:teuthology.orchestra.run.smithi028.stdout:mon.smithi144 smithi144 running (2m) 66s ago 12m 38.1M 2048M 19.3.0-4917-g8293d73f 425e316ba8eb c081c5ed44b5 2024-09-16T08:20:10.846 INFO:teuthology.orchestra.run.smithi028.stdout:nfs.foo.0.0.smithi028.dkttgj smithi028 *:2049 running (7m) 28s ago 7m 282M - 4.0 e1d6a67b021e b5dccc45a6ed 2024-09-16T08:20:10.846 INFO:teuthology.orchestra.run.smithi028.stdout:nfs.foo.1.0.smithi144.epipkg smithi144 *:2049 running (7m) 66s ago 7m 87.0M - 4.0 e1d6a67b021e faa89e61f50e 2024-09-16T08:20:10.846 INFO:teuthology.orchestra.run.smithi028.stdout:node-exporter.smithi028 smithi028 *:9100 running (4m) 28s ago 13m 9949k - 1.7.0 72c9c2088986 9e2b8e8a63a9 2024-09-16T08:20:10.846 INFO:teuthology.orchestra.run.smithi028.stdout:node-exporter.smithi144 smithi144 *:9100 running (3m) 66s ago 12m 10.1M - 1.7.0 72c9c2088986 78834c5e992f 2024-09-16T08:20:10.846 INFO:teuthology.orchestra.run.smithi028.stdout:osd.0 smithi028 running (37s) 28s ago 12m 14.4M 1937M 19.3.0-4917-g8293d73f 425e316ba8eb 2fc02ba4f14f 2024-09-16T08:20:10.846 INFO:teuthology.orchestra.run.smithi028.stdout:osd.1 smithi028 running (11m) 28s ago 11m 525M 1937M 17.2.0 e1d6a67b021e 6911d5c68d54 2024-09-16T08:20:10.846 INFO:teuthology.orchestra.run.smithi028.stdout:osd.2 smithi028 running (11m) 28s ago 11m 417M 1937M 17.2.0 e1d6a67b021e 90b27d8375a2 2024-09-16T08:20:10.846 INFO:teuthology.orchestra.run.smithi028.stdout:osd.3 smithi028 running (10m) 28s ago 10m 537M 1937M 17.2.0 e1d6a67b021e dac477d8b04f 2024-09-16T08:20:10.846 INFO:teuthology.orchestra.run.smithi028.stdout:osd.4 smithi144 running (10m) 66s ago 10m 527M 2704M 17.2.0 e1d6a67b021e 86719c60975c 2024-09-16T08:20:10.846 INFO:teuthology.orchestra.run.smithi028.stdout:osd.5 smithi144 running (10m) 66s ago 10m 334M 2704M 17.2.0 e1d6a67b021e 70452c145c60 2024-09-16T08:20:10.847 INFO:teuthology.orchestra.run.smithi028.stdout:osd.6 smithi144 running (9m) 66s ago 9m 533M 2704M 17.2.0 e1d6a67b021e 7f9981be792f 2024-09-16T08:20:10.847 INFO:teuthology.orchestra.run.smithi028.stdout:osd.7 smithi144 running (9m) 66s ago 9m 429M 2704M 17.2.0 e1d6a67b021e dada74ddbe4c 2024-09-16T08:20:10.847 INFO:teuthology.orchestra.run.smithi028.stdout:prometheus.smithi028 smithi028 *:9095 running (2m) 28s ago 12m 50.7M - 2.51.0 1d3b7f56885b 42667809df2b 2024-09-16T08:20:11.053 INFO:teuthology.orchestra.run.smithi028.stdout: 5 186787 10.26 MB/sec execute 309 sec latency 84.095 ms 2024-09-16T08:20:11.055 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:20:10 smithi028 ceph-mon[64115]: from='client.34232 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T08:20:11.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:20:10 smithi144 ceph-mon[49149]: from='client.34232 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T08:20:11.550 INFO:teuthology.orchestra.run.smithi028.stdout:{ 2024-09-16T08:20:11.550 INFO:teuthology.orchestra.run.smithi028.stdout: "mon": { 2024-09-16T08:20:11.551 INFO:teuthology.orchestra.run.smithi028.stdout: "ceph version 19.3.0-4917-g8293d73f (8293d73f8690540e843a81caec373f9cc29cf705) squid (dev)": 2 2024-09-16T08:20:11.551 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-09-16T08:20:11.551 INFO:teuthology.orchestra.run.smithi028.stdout: "mgr": { 2024-09-16T08:20:11.551 INFO:teuthology.orchestra.run.smithi028.stdout: "ceph version 19.3.0-4917-g8293d73f (8293d73f8690540e843a81caec373f9cc29cf705) squid (dev)": 2 2024-09-16T08:20:11.551 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-09-16T08:20:11.551 INFO:teuthology.orchestra.run.smithi028.stdout: "osd": { 2024-09-16T08:20:11.552 INFO:teuthology.orchestra.run.smithi028.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 7, 2024-09-16T08:20:11.552 INFO:teuthology.orchestra.run.smithi028.stdout: "ceph version 19.3.0-4917-g8293d73f (8293d73f8690540e843a81caec373f9cc29cf705) squid (dev)": 1 2024-09-16T08:20:11.552 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-09-16T08:20:11.552 INFO:teuthology.orchestra.run.smithi028.stdout: "mds": { 2024-09-16T08:20:11.552 INFO:teuthology.orchestra.run.smithi028.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-16T08:20:11.552 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-09-16T08:20:11.553 INFO:teuthology.orchestra.run.smithi028.stdout: "overall": { 2024-09-16T08:20:11.553 INFO:teuthology.orchestra.run.smithi028.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 9, 2024-09-16T08:20:11.553 INFO:teuthology.orchestra.run.smithi028.stdout: "ceph version 19.3.0-4917-g8293d73f (8293d73f8690540e843a81caec373f9cc29cf705) squid (dev)": 5 2024-09-16T08:20:11.553 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-09-16T08:20:11.553 INFO:teuthology.orchestra.run.smithi028.stdout:} 2024-09-16T08:20:12.053 INFO:teuthology.orchestra.run.smithi028.stdout: 5 187161 10.26 MB/sec execute 310 sec latency 62.146 ms 2024-09-16T08:20:12.082 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:20:11 smithi028 ceph-mon[64115]: from='client.34234 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T08:20:12.082 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:20:11 smithi028 ceph-mon[64115]: pgmap v85: 97 pgs: 97 active+clean; 350 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 7.5 KiB/s rd, 6.3 MiB/s wr, 588 op/s 2024-09-16T08:20:12.082 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:20:11 smithi028 ceph-mon[64115]: from='client.34238 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T08:20:12.082 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:20:11 smithi028 ceph-mon[64115]: from='client.? 172.21.15.28:0/3234193548' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-16T08:20:12.123 INFO:teuthology.orchestra.run.smithi028.stdout:{ 2024-09-16T08:20:12.123 INFO:teuthology.orchestra.run.smithi028.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:f7ad55f9ccd17ba66aadb19b4d291e67b60ea24e5d8a0c0e92941fd59c81e4ce", 2024-09-16T08:20:12.123 INFO:teuthology.orchestra.run.smithi028.stdout: "in_progress": true, 2024-09-16T08:20:12.123 INFO:teuthology.orchestra.run.smithi028.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-16T08:20:12.124 INFO:teuthology.orchestra.run.smithi028.stdout: "services_complete": [ 2024-09-16T08:20:12.124 INFO:teuthology.orchestra.run.smithi028.stdout: "crash", 2024-09-16T08:20:12.124 INFO:teuthology.orchestra.run.smithi028.stdout: "mon", 2024-09-16T08:20:12.124 INFO:teuthology.orchestra.run.smithi028.stdout: "mgr" 2024-09-16T08:20:12.124 INFO:teuthology.orchestra.run.smithi028.stdout: ], 2024-09-16T08:20:12.124 INFO:teuthology.orchestra.run.smithi028.stdout: "progress": "7/23 daemons upgraded", 2024-09-16T08:20:12.124 INFO:teuthology.orchestra.run.smithi028.stdout: "message": "Currently upgrading osd daemons", 2024-09-16T08:20:12.124 INFO:teuthology.orchestra.run.smithi028.stdout: "is_paused": false 2024-09-16T08:20:12.124 INFO:teuthology.orchestra.run.smithi028.stdout:} 2024-09-16T08:20:12.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:20:11 smithi144 ceph-mon[49149]: from='client.34234 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T08:20:12.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:20:11 smithi144 ceph-mon[49149]: pgmap v85: 97 pgs: 97 active+clean; 350 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 7.5 KiB/s rd, 6.3 MiB/s wr, 588 op/s 2024-09-16T08:20:12.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:20:11 smithi144 ceph-mon[49149]: from='client.34238 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T08:20:12.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:20:11 smithi144 ceph-mon[49149]: from='client.? 172.21.15.28:0/3234193548' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-16T08:20:12.587 INFO:journalctl@ceph.osd.1.smithi028.stdout:Sep 16 08:20:12 smithi028 systemd[1]: Stopping Ceph osd.1 for 5622b2fe-7402-11ef-bceb-c7b262605968... 2024-09-16T08:20:12.726 INFO:teuthology.orchestra.run.smithi028.stdout:HEALTH_OK 2024-09-16T08:20:12.838 INFO:journalctl@ceph.osd.1.smithi028.stdout:Sep 16 08:20:12 smithi028 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-1[37067]: 2024-09-16T08:20:12.758+0000 7f586bda3700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.1 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-16T08:20:12.838 INFO:journalctl@ceph.osd.1.smithi028.stdout:Sep 16 08:20:12 smithi028 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-1[37067]: 2024-09-16T08:20:12.758+0000 7f586bda3700 -1 osd.1 82 *** Got signal Terminated *** 2024-09-16T08:20:12.838 INFO:journalctl@ceph.osd.1.smithi028.stdout:Sep 16 08:20:12 smithi028 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-1[37067]: 2024-09-16T08:20:12.758+0000 7f586bda3700 -1 osd.1 82 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-16T08:20:13.053 INFO:teuthology.orchestra.run.smithi028.stdout: 5 187483 10.26 MB/sec execute 311 sec latency 66.955 ms 2024-09-16T08:20:13.093 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:20:12 smithi028 ceph-mon[64115]: from='client.44187 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T08:20:13.093 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:20:12 smithi028 ceph-mon[64115]: from='client.? 172.21.15.28:0/3477194454' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-16T08:20:13.093 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:20:12 smithi028 ceph-mon[64115]: osd.1 marked itself down and dead 2024-09-16T08:20:13.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:20:12 smithi144 ceph-mon[49149]: from='client.44187 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T08:20:13.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:20:12 smithi144 ceph-mon[49149]: from='client.? 172.21.15.28:0/3477194454' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-16T08:20:13.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:20:12 smithi144 ceph-mon[49149]: osd.1 marked itself down and dead 2024-09-16T08:20:13.826 INFO:journalctl@ceph.osd.1.smithi028.stdout:Sep 16 08:20:13 smithi028 podman[70885]: 2024-09-16 08:20:13.572932496 +0000 UTC m=+0.997432377 container died 6911d5c68d54dd89cc443bfd0b6bb73d89cf6ab9cbfc3c40210169b9b3e39daa (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-1, ceph=True, GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.component=centos-stream-container, RELEASE=HEAD, GIT_BRANCH=HEAD, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.expose-services=, release=754, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vcs-type=git, distribution-scope=public, GIT_CLEAN=True, io.openshift.tags=base centos centos-stream, architecture=x86_64, name=centos-stream, io.k8s.display-name=CentOS Stream 8, maintainer=Guillaume Abrioux , version=8, CEPH_POINT_RELEASE=-17.2.0, io.buildah.version=1.19.8, vendor=Red Hat, Inc., build-date=2022-05-03T08:36:31.336870) 2024-09-16T08:20:14.053 INFO:teuthology.orchestra.run.smithi028.stdout: 5 187587 10.24 MB/sec execute 312 sec latency 961.933 ms 2024-09-16T08:20:14.078 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:20:13 smithi028 ceph-mon[64115]: pgmap v86: 97 pgs: 97 active+clean; 354 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 6.4 MiB/s wr, 650 op/s 2024-09-16T08:20:14.078 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:20:13 smithi028 ceph-mon[64115]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-16T08:20:14.078 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:20:13 smithi028 ceph-mon[64115]: osdmap e83: 8 total, 7 up, 8 in 2024-09-16T08:20:14.078 INFO:journalctl@ceph.osd.1.smithi028.stdout:Sep 16 08:20:13 smithi028 podman[70885]: 2024-09-16 08:20:13.917229251 +0000 UTC m=+1.341729125 container cleanup 6911d5c68d54dd89cc443bfd0b6bb73d89cf6ab9cbfc3c40210169b9b3e39daa (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-1, ceph=True, io.k8s.display-name=CentOS Stream 8, release=754, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, name=centos-stream, architecture=x86_64, version=8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_REPO=https://github.com/ceph/ceph-container.git, build-date=2022-05-03T08:36:31.336870, io.buildah.version=1.19.8, maintainer=Guillaume Abrioux , com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.openshift.expose-services=, GIT_CLEAN=True, vendor=Red Hat, Inc., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vcs-type=git, CEPH_POINT_RELEASE=-17.2.0, GIT_BRANCH=HEAD, com.redhat.component=centos-stream-container, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, RELEASE=HEAD, io.openshift.tags=base centos centos-stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, distribution-scope=public, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly.) 2024-09-16T08:20:14.078 INFO:journalctl@ceph.osd.1.smithi028.stdout:Sep 16 08:20:13 smithi028 bash[70885]: ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-1 2024-09-16T08:20:14.329 INFO:journalctl@ceph.osd.1.smithi028.stdout:Sep 16 08:20:14 smithi028 podman[70905]: 2024-09-16 08:20:14.303477704 +0000 UTC m=+0.733294835 container remove 6911d5c68d54dd89cc443bfd0b6bb73d89cf6ab9cbfc3c40210169b9b3e39daa (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-1, io.openshift.expose-services=, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, build-date=2022-05-03T08:36:31.336870, io.buildah.version=1.19.8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, name=centos-stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vendor=Red Hat, Inc., GIT_CLEAN=True, architecture=x86_64, RELEASE=HEAD, ceph=True, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, CEPH_POINT_RELEASE=-17.2.0, io.k8s.display-name=CentOS Stream 8, vcs-type=git, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_REPO=https://github.com/ceph/ceph-container.git, maintainer=Guillaume Abrioux , com.redhat.license_terms=https://centos.org/legal/licensing-policy/, com.redhat.component=centos-stream-container, io.openshift.tags=base centos centos-stream, distribution-scope=public, GIT_BRANCH=HEAD, version=8, release=754) 2024-09-16T08:20:14.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:20:13 smithi144 ceph-mon[49149]: pgmap v86: 97 pgs: 97 active+clean; 354 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 6.4 MiB/s wr, 650 op/s 2024-09-16T08:20:14.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:20:13 smithi144 ceph-mon[49149]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-16T08:20:14.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:20:13 smithi144 ceph-mon[49149]: osdmap e83: 8 total, 7 up, 8 in 2024-09-16T08:20:14.831 INFO:journalctl@ceph.osd.1.smithi028.stdout:Sep 16 08:20:14 smithi028 podman[70989]: 2024-09-16 08:20:14.626224861 +0000 UTC m=+0.026050707 image pull 425e316ba8ebf74f04aa3640f7deb4fa3f293a78d0c8d097cb679eaf86ddca6c quay.ceph.io/ceph-ci/ceph@sha256:f7ad55f9ccd17ba66aadb19b4d291e67b60ea24e5d8a0c0e92941fd59c81e4ce 2024-09-16T08:20:14.831 INFO:journalctl@ceph.osd.1.smithi028.stdout:Sep 16 08:20:14 smithi028 podman[70989]: 2024-09-16 08:20:14.746426621 +0000 UTC m=+0.146252462 container create 6c8fb20d130996e313c748c0b02bd4b61e2d194b36f58f653771f7a9f15d9429 (image=quay.ceph.io/ceph-ci/ceph@sha256:f7ad55f9ccd17ba66aadb19b4d291e67b60ea24e5d8a0c0e92941fd59c81e4ce, name=ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-1-deactivate, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, RELEASE=wip-guits-main-2024-09-13-1248-8293d73, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.schema-version=1.0, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-16T08:20:15.054 INFO:teuthology.orchestra.run.smithi028.stdout: 5 187989 10.25 MB/sec execute 313 sec latency 66.284 ms 2024-09-16T08:20:15.082 INFO:journalctl@ceph.osd.1.smithi028.stdout:Sep 16 08:20:15 smithi028 podman[70989]: 2024-09-16 08:20:15.058808559 +0000 UTC m=+0.458634403 container init 6c8fb20d130996e313c748c0b02bd4b61e2d194b36f58f653771f7a9f15d9429 (image=quay.ceph.io/ceph-ci/ceph@sha256:f7ad55f9ccd17ba66aadb19b4d291e67b60ea24e5d8a0c0e92941fd59c81e4ce, name=ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-1-deactivate, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, ceph=True, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, RELEASE=wip-guits-main-2024-09-13-1248-8293d73, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2) 2024-09-16T08:20:15.082 INFO:journalctl@ceph.osd.1.smithi028.stdout:Sep 16 08:20:15 smithi028 podman[70989]: 2024-09-16 08:20:15.068433247 +0000 UTC m=+0.468259083 container start 6c8fb20d130996e313c748c0b02bd4b61e2d194b36f58f653771f7a9f15d9429 (image=quay.ceph.io/ceph-ci/ceph@sha256:f7ad55f9ccd17ba66aadb19b4d291e67b60ea24e5d8a0c0e92941fd59c81e4ce, name=ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-1-deactivate, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=wip-guits-main-2024-09-13-1248-8293d73, org.label-schema.license=GPLv2, ceph=True, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2) 2024-09-16T08:20:15.082 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:20:14 smithi028 ceph-mon[64115]: osdmap e84: 8 total, 7 up, 8 in 2024-09-16T08:20:15.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:20:14 smithi144 ceph-mon[49149]: osdmap e84: 8 total, 7 up, 8 in 2024-09-16T08:20:15.336 INFO:journalctl@ceph.osd.1.smithi028.stdout:Sep 16 08:20:15 smithi028 podman[70989]: 2024-09-16 08:20:15.219417375 +0000 UTC m=+0.619243213 container attach 6c8fb20d130996e313c748c0b02bd4b61e2d194b36f58f653771f7a9f15d9429 (image=quay.ceph.io/ceph-ci/ceph@sha256:f7ad55f9ccd17ba66aadb19b4d291e67b60ea24e5d8a0c0e92941fd59c81e4ce, name=ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-1-deactivate, org.label-schema.build-date=20240731, GIT_CLEAN=True, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=wip-guits-main-2024-09-13-1248-8293d73, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS) 2024-09-16T08:20:15.598 INFO:journalctl@ceph.osd.1.smithi028.stdout:Sep 16 08:20:15 smithi028 podman[70989]: 2024-09-16 08:20:15.407394654 +0000 UTC m=+0.807220499 container died 6c8fb20d130996e313c748c0b02bd4b61e2d194b36f58f653771f7a9f15d9429 (image=quay.ceph.io/ceph-ci/ceph@sha256:f7ad55f9ccd17ba66aadb19b4d291e67b60ea24e5d8a0c0e92941fd59c81e4ce, name=ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-1-deactivate, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=wip-guits-main-2024-09-13-1248-8293d73, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0) 2024-09-16T08:20:16.054 INFO:teuthology.orchestra.run.smithi028.stdout: 5 188290 10.25 MB/sec execute 314 sec latency 123.619 ms 2024-09-16T08:20:16.110 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:20:15 smithi028 ceph-mon[64115]: pgmap v89: 97 pgs: 12 stale+active+clean, 85 active+clean; 346 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 9.9 KiB/s rd, 6.4 MiB/s wr, 646 op/s 2024-09-16T08:20:16.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:20:15 smithi144 ceph-mon[49149]: pgmap v89: 97 pgs: 12 stale+active+clean, 85 active+clean; 346 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 9.9 KiB/s rd, 6.4 MiB/s wr, 646 op/s 2024-09-16T08:20:16.631 INFO:journalctl@ceph.osd.1.smithi028.stdout:Sep 16 08:20:16 smithi028 podman[71012]: 2024-09-16 08:20:16.396984594 +0000 UTC m=+0.981402029 container remove 6c8fb20d130996e313c748c0b02bd4b61e2d194b36f58f653771f7a9f15d9429 (image=quay.ceph.io/ceph-ci/ceph@sha256:f7ad55f9ccd17ba66aadb19b4d291e67b60ea24e5d8a0c0e92941fd59c81e4ce, name=ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-1-deactivate, RELEASE=wip-guits-main-2024-09-13-1248-8293d73, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_CLEAN=True, ceph=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-16T08:20:16.631 INFO:journalctl@ceph.osd.1.smithi028.stdout:Sep 16 08:20:16 smithi028 systemd[1]: ceph-5622b2fe-7402-11ef-bceb-c7b262605968@osd.1.service: Deactivated successfully. 2024-09-16T08:20:16.632 INFO:journalctl@ceph.osd.1.smithi028.stdout:Sep 16 08:20:16 smithi028 systemd[1]: Stopped Ceph osd.1 for 5622b2fe-7402-11ef-bceb-c7b262605968. 2024-09-16T08:20:16.632 INFO:journalctl@ceph.osd.1.smithi028.stdout:Sep 16 08:20:16 smithi028 systemd[1]: ceph-5622b2fe-7402-11ef-bceb-c7b262605968@osd.1.service: Consumed 3min 23.285s CPU time. 2024-09-16T08:20:16.890 INFO:journalctl@ceph.osd.1.smithi028.stdout:Sep 16 08:20:16 smithi028 systemd[1]: Starting Ceph osd.1 for 5622b2fe-7402-11ef-bceb-c7b262605968... 2024-09-16T08:20:17.054 INFO:teuthology.orchestra.run.smithi028.stdout: 5 188541 10.24 MB/sec execute 315 sec latency 78.169 ms 2024-09-16T08:20:17.148 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:20:17 smithi028 ceph-mon[64115]: Health check failed: Degraded data redundancy: 13/1632 objects degraded (0.797%), 2 pgs degraded (PG_DEGRADED) 2024-09-16T08:20:17.148 INFO:journalctl@ceph.osd.1.smithi028.stdout:Sep 16 08:20:16 smithi028 podman[71107]: 2024-09-16 08:20:16.871387522 +0000 UTC m=+0.020396604 image pull 425e316ba8ebf74f04aa3640f7deb4fa3f293a78d0c8d097cb679eaf86ddca6c quay.ceph.io/ceph-ci/ceph@sha256:f7ad55f9ccd17ba66aadb19b4d291e67b60ea24e5d8a0c0e92941fd59c81e4ce 2024-09-16T08:20:17.148 INFO:journalctl@ceph.osd.1.smithi028.stdout:Sep 16 08:20:17 smithi028 podman[71107]: 2024-09-16 08:20:17.028135179 +0000 UTC m=+0.177144254 container create 2a6e491f61705b56c2e8c6b9317abc76ce2242e6cf48aabc2a6b728c18642365 (image=quay.ceph.io/ceph-ci/ceph@sha256:f7ad55f9ccd17ba66aadb19b4d291e67b60ea24e5d8a0c0e92941fd59c81e4ce, name=ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-1-activate, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, RELEASE=wip-guits-main-2024-09-13-1248-8293d73, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_CLEAN=True, ceph=True, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD) 2024-09-16T08:20:17.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:20:17 smithi144 ceph-mon[49149]: Health check failed: Degraded data redundancy: 13/1632 objects degraded (0.797%), 2 pgs degraded (PG_DEGRADED) 2024-09-16T08:20:17.402 INFO:journalctl@ceph.osd.1.smithi028.stdout:Sep 16 08:20:17 smithi028 podman[71107]: 2024-09-16 08:20:17.390835998 +0000 UTC m=+0.539845072 container init 2a6e491f61705b56c2e8c6b9317abc76ce2242e6cf48aabc2a6b728c18642365 (image=quay.ceph.io/ceph-ci/ceph@sha256:f7ad55f9ccd17ba66aadb19b4d291e67b60ea24e5d8a0c0e92941fd59c81e4ce, name=ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-1-activate, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, ceph=True, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-guits-main-2024-09-13-1248-8293d73, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965) 2024-09-16T08:20:17.661 INFO:journalctl@ceph.osd.1.smithi028.stdout:Sep 16 08:20:17 smithi028 podman[71107]: 2024-09-16 08:20:17.401849141 +0000 UTC m=+0.550858215 container start 2a6e491f61705b56c2e8c6b9317abc76ce2242e6cf48aabc2a6b728c18642365 (image=quay.ceph.io/ceph-ci/ceph@sha256:f7ad55f9ccd17ba66aadb19b4d291e67b60ea24e5d8a0c0e92941fd59c81e4ce, name=ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-1-activate, ceph=True, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.build-date=20240731, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, RELEASE=wip-guits-main-2024-09-13-1248-8293d73) 2024-09-16T08:20:17.661 INFO:journalctl@ceph.osd.1.smithi028.stdout:Sep 16 08:20:17 smithi028 podman[71107]: 2024-09-16 08:20:17.544259832 +0000 UTC m=+0.693268915 container attach 2a6e491f61705b56c2e8c6b9317abc76ce2242e6cf48aabc2a6b728c18642365 (image=quay.ceph.io/ceph-ci/ceph@sha256:f7ad55f9ccd17ba66aadb19b4d291e67b60ea24e5d8a0c0e92941fd59c81e4ce, name=ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-1-activate, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_CLEAN=True, RELEASE=wip-guits-main-2024-09-13-1248-8293d73, io.buildah.version=1.37.2, ceph=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_BRANCH=HEAD) 2024-09-16T08:20:17.661 INFO:journalctl@ceph.osd.1.smithi028.stdout:Sep 16 08:20:17 smithi028 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-1-activate[71137]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-16T08:20:17.661 INFO:journalctl@ceph.osd.1.smithi028.stdout:Sep 16 08:20:17 smithi028 bash[71107]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-16T08:20:17.661 INFO:journalctl@ceph.osd.1.smithi028.stdout:Sep 16 08:20:17 smithi028 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-1-activate[71137]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-16T08:20:17.661 INFO:journalctl@ceph.osd.1.smithi028.stdout:Sep 16 08:20:17 smithi028 bash[71107]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-16T08:20:18.054 INFO:teuthology.orchestra.run.smithi028.stdout: 5 188763 10.22 MB/sec execute 316 sec latency 68.562 ms 2024-09-16T08:20:18.164 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:20:18 smithi028 ceph-mon[64115]: pgmap v90: 97 pgs: 2 active+undersized, 11 stale+active+clean, 2 active+undersized+degraded, 82 active+clean; 346 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 3.2 MiB/s wr, 328 op/s; 13/1632 objects degraded (0.797%) 2024-09-16T08:20:18.164 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:20:18 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:20:18.164 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:20:18 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-16T08:20:18.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:20:18 smithi144 ceph-mon[49149]: pgmap v90: 97 pgs: 2 active+undersized, 11 stale+active+clean, 2 active+undersized+degraded, 82 active+clean; 346 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 3.2 MiB/s wr, 328 op/s; 13/1632 objects degraded (0.797%) 2024-09-16T08:20:18.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:20:18 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:20:18.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:20:18 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-16T08:20:19.054 INFO:teuthology.orchestra.run.smithi028.stdout: 5 189024 10.21 MB/sec execute 317 sec latency 72.926 ms 2024-09-16T08:20:20.054 INFO:teuthology.orchestra.run.smithi028.stdout: 5 189367 10.22 MB/sec execute 318 sec latency 43.102 ms 2024-09-16T08:20:20.199 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:20:20 smithi028 ceph-mon[64115]: pgmap v91: 97 pgs: 15 active+undersized, 24 active+undersized+degraded, 58 active+clean; 346 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 9.4 KiB/s rd, 5.0 MiB/s wr, 516 op/s; 169/1635 objects degraded (10.336%) 2024-09-16T08:20:20.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:20:20 smithi144 ceph-mon[49149]: pgmap v91: 97 pgs: 15 active+undersized, 24 active+undersized+degraded, 58 active+clean; 346 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 9.4 KiB/s rd, 5.0 MiB/s wr, 516 op/s; 169/1635 objects degraded (10.336%) 2024-09-16T08:20:20.704 INFO:journalctl@ceph.osd.1.smithi028.stdout:Sep 16 08:20:20 smithi028 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-1-activate[71137]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-09-16T08:20:20.704 INFO:journalctl@ceph.osd.1.smithi028.stdout:Sep 16 08:20:20 smithi028 bash[71107]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-09-16T08:20:20.704 INFO:journalctl@ceph.osd.1.smithi028.stdout:Sep 16 08:20:20 smithi028 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-1-activate[71137]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-1 --no-mon-config --dev /dev/mapper/vg_nvme-lv_3 2024-09-16T08:20:20.704 INFO:journalctl@ceph.osd.1.smithi028.stdout:Sep 16 08:20:20 smithi028 bash[71107]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-1 --no-mon-config --dev /dev/mapper/vg_nvme-lv_3 2024-09-16T08:20:20.957 INFO:journalctl@ceph.osd.1.smithi028.stdout:Sep 16 08:20:20 smithi028 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-1-activate[71137]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_3 2024-09-16T08:20:20.957 INFO:journalctl@ceph.osd.1.smithi028.stdout:Sep 16 08:20:20 smithi028 bash[71107]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_3 2024-09-16T08:20:20.957 INFO:journalctl@ceph.osd.1.smithi028.stdout:Sep 16 08:20:20 smithi028 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-1-activate[71137]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-09-16T08:20:20.957 INFO:journalctl@ceph.osd.1.smithi028.stdout:Sep 16 08:20:20 smithi028 bash[71107]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-09-16T08:20:20.957 INFO:journalctl@ceph.osd.1.smithi028.stdout:Sep 16 08:20:20 smithi028 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-1-activate[71137]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-1/block 2024-09-16T08:20:20.958 INFO:journalctl@ceph.osd.1.smithi028.stdout:Sep 16 08:20:20 smithi028 bash[71107]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-1/block 2024-09-16T08:20:20.958 INFO:journalctl@ceph.osd.1.smithi028.stdout:Sep 16 08:20:20 smithi028 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-1-activate[71137]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-09-16T08:20:20.958 INFO:journalctl@ceph.osd.1.smithi028.stdout:Sep 16 08:20:20 smithi028 bash[71107]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-09-16T08:20:20.958 INFO:journalctl@ceph.osd.1.smithi028.stdout:Sep 16 08:20:20 smithi028 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-1-activate[71137]: --> ceph-volume raw activate successful for osd ID: 1 2024-09-16T08:20:20.958 INFO:journalctl@ceph.osd.1.smithi028.stdout:Sep 16 08:20:20 smithi028 bash[71107]: --> ceph-volume raw activate successful for osd ID: 1 2024-09-16T08:20:21.054 INFO:teuthology.orchestra.run.smithi028.stdout: 5 189687 10.21 MB/sec execute 319 sec latency 58.494 ms 2024-09-16T08:20:21.210 INFO:journalctl@ceph.osd.1.smithi028.stdout:Sep 16 08:20:21 smithi028 podman[71292]: 2024-09-16 08:20:21.021000329 +0000 UTC m=+0.025620638 container died 2a6e491f61705b56c2e8c6b9317abc76ce2242e6cf48aabc2a6b728c18642365 (image=quay.ceph.io/ceph-ci/ceph@sha256:f7ad55f9ccd17ba66aadb19b4d291e67b60ea24e5d8a0c0e92941fd59c81e4ce, name=ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-1-activate, CEPH_POINT_RELEASE=, GIT_CLEAN=True, RELEASE=wip-guits-main-2024-09-13-1248-8293d73, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, ceph=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-16T08:20:21.973 INFO:journalctl@ceph.osd.1.smithi028.stdout:Sep 16 08:20:21 smithi028 podman[71292]: 2024-09-16 08:20:21.891278495 +0000 UTC m=+0.895898798 container remove 2a6e491f61705b56c2e8c6b9317abc76ce2242e6cf48aabc2a6b728c18642365 (image=quay.ceph.io/ceph-ci/ceph@sha256:f7ad55f9ccd17ba66aadb19b4d291e67b60ea24e5d8a0c0e92941fd59c81e4ce, name=ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-1-activate, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, RELEASE=wip-guits-main-2024-09-13-1248-8293d73, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.license=GPLv2, ceph=True, maintainer=Guillaume Abrioux , GIT_CLEAN=True) 2024-09-16T08:20:22.054 INFO:teuthology.orchestra.run.smithi028.stdout: 5 190004 10.20 MB/sec execute 320 sec latency 61.024 ms 2024-09-16T08:20:22.228 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:20:22 smithi028 ceph-mon[64115]: pgmap v92: 97 pgs: 15 active+undersized, 24 active+undersized+degraded, 58 active+clean; 346 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 3.6 KiB/s rd, 4.4 MiB/s wr, 405 op/s; 171/1644 objects degraded (10.401%) 2024-09-16T08:20:22.228 INFO:journalctl@ceph.osd.1.smithi028.stdout:Sep 16 08:20:22 smithi028 podman[71347]: 2024-09-16 08:20:22.026188746 +0000 UTC m=+0.018795698 image pull 425e316ba8ebf74f04aa3640f7deb4fa3f293a78d0c8d097cb679eaf86ddca6c quay.ceph.io/ceph-ci/ceph@sha256:f7ad55f9ccd17ba66aadb19b4d291e67b60ea24e5d8a0c0e92941fd59c81e4ce 2024-09-16T08:20:22.228 INFO:journalctl@ceph.osd.1.smithi028.stdout:Sep 16 08:20:22 smithi028 podman[71347]: 2024-09-16 08:20:22.134226841 +0000 UTC m=+0.126833788 container create df34bd603d8f561da8f37f5dc13a453f49ffdedf58bdeb0658d1cca697dae0f2 (image=quay.ceph.io/ceph-ci/ceph@sha256:f7ad55f9ccd17ba66aadb19b4d291e67b60ea24e5d8a0c0e92941fd59c81e4ce, name=ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-1, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=wip-guits-main-2024-09-13-1248-8293d73, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, GIT_CLEAN=True, io.buildah.version=1.37.2, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0) 2024-09-16T08:20:22.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:20:22 smithi144 ceph-mon[49149]: pgmap v92: 97 pgs: 15 active+undersized, 24 active+undersized+degraded, 58 active+clean; 346 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 3.6 KiB/s rd, 4.4 MiB/s wr, 405 op/s; 171/1644 objects degraded (10.401%) 2024-09-16T08:20:22.481 INFO:journalctl@ceph.osd.1.smithi028.stdout:Sep 16 08:20:22 smithi028 podman[71347]: 2024-09-16 08:20:22.378487873 +0000 UTC m=+0.371094833 container init df34bd603d8f561da8f37f5dc13a453f49ffdedf58bdeb0658d1cca697dae0f2 (image=quay.ceph.io/ceph-ci/ceph@sha256:f7ad55f9ccd17ba66aadb19b4d291e67b60ea24e5d8a0c0e92941fd59c81e4ce, name=ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-1, CEPH_POINT_RELEASE=, RELEASE=wip-guits-main-2024-09-13-1248-8293d73, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_CLEAN=True, io.buildah.version=1.37.2, GIT_BRANCH=HEAD) 2024-09-16T08:20:22.481 INFO:journalctl@ceph.osd.1.smithi028.stdout:Sep 16 08:20:22 smithi028 podman[71347]: 2024-09-16 08:20:22.389804352 +0000 UTC m=+0.382411302 container start df34bd603d8f561da8f37f5dc13a453f49ffdedf58bdeb0658d1cca697dae0f2 (image=quay.ceph.io/ceph-ci/ceph@sha256:f7ad55f9ccd17ba66aadb19b4d291e67b60ea24e5d8a0c0e92941fd59c81e4ce, name=ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-1, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, RELEASE=wip-guits-main-2024-09-13-1248-8293d73, CEPH_POINT_RELEASE=, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, org.label-schema.build-date=20240731) 2024-09-16T08:20:22.481 INFO:journalctl@ceph.osd.1.smithi028.stdout:Sep 16 08:20:22 smithi028 ceph-osd[71365]: -- 172.21.15.28:0/1144376908 <== mon.0 v2:172.21.15.28:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x555f1b7ae1e0 con 0x555f1a94dc00 2024-09-16T08:20:22.481 INFO:journalctl@ceph.osd.1.smithi028.stdout:Sep 16 08:20:22 smithi028 bash[71347]: df34bd603d8f561da8f37f5dc13a453f49ffdedf58bdeb0658d1cca697dae0f2 2024-09-16T08:20:22.734 INFO:journalctl@ceph.osd.1.smithi028.stdout:Sep 16 08:20:22 smithi028 systemd[1]: Started Ceph osd.1 for 5622b2fe-7402-11ef-bceb-c7b262605968. 2024-09-16T08:20:23.054 INFO:teuthology.orchestra.run.smithi028.stdout: 5 190258 10.18 MB/sec execute 321 sec latency 64.760 ms 2024-09-16T08:20:23.647 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:20:23 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:20:23.648 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:20:23 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:20:23.648 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:20:23 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-16T08:20:23.648 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:20:23 smithi028 ceph-mon[64115]: pgmap v93: 97 pgs: 15 active+undersized, 24 active+undersized+degraded, 58 active+clean; 345 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 6.9 KiB/s rd, 2.8 MiB/s wr, 218 op/s; 178/1659 objects degraded (10.729%) 2024-09-16T08:20:23.832 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:20:23 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:20:23.832 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:20:23 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:20:23.832 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:20:23 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-16T08:20:23.833 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:20:23 smithi144 ceph-mon[49149]: pgmap v93: 97 pgs: 15 active+undersized, 24 active+undersized+degraded, 58 active+clean; 345 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 6.9 KiB/s rd, 2.8 MiB/s wr, 218 op/s; 178/1659 objects degraded (10.729%) 2024-09-16T08:20:24.054 INFO:teuthology.orchestra.run.smithi028.stdout: 5 190586 10.17 MB/sec execute 322 sec latency 60.859 ms 2024-09-16T08:20:24.690 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:20:24 smithi028 ceph-mon[64115]: Health check update: Degraded data redundancy: 178/1659 objects degraded (10.729%), 24 pgs degraded (PG_DEGRADED) 2024-09-16T08:20:24.831 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:20:24 smithi144 ceph-mon[49149]: Health check update: Degraded data redundancy: 178/1659 objects degraded (10.729%), 24 pgs degraded (PG_DEGRADED) 2024-09-16T08:20:24.947 INFO:journalctl@ceph.osd.1.smithi028.stdout:Sep 16 08:20:24 smithi028 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-1[71361]: 2024-09-16T08:20:24.758+0000 7f6be3992640 -1 Falling back to public interface 2024-09-16T08:20:25.056 INFO:teuthology.orchestra.run.smithi028.stdout: 5 190813 10.16 MB/sec execute 323 sec latency 39.252 ms 2024-09-16T08:20:25.740 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:20:25 smithi028 ceph-mon[64115]: pgmap v94: 97 pgs: 15 active+undersized, 24 active+undersized+degraded, 58 active+clean; 344 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 3.8 MiB/s wr, 377 op/s; 175/1629 objects degraded (10.743%) 2024-09-16T08:20:25.831 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:20:25 smithi144 ceph-mon[49149]: pgmap v94: 97 pgs: 15 active+undersized, 24 active+undersized+degraded, 58 active+clean; 344 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 3.8 MiB/s wr, 377 op/s; 175/1629 objects degraded (10.743%) 2024-09-16T08:20:26.054 INFO:teuthology.orchestra.run.smithi028.stdout: 5 191159 10.17 MB/sec execute 324 sec latency 67.639 ms 2024-09-16T08:20:27.054 INFO:teuthology.orchestra.run.smithi028.stdout: 5 191502 10.17 MB/sec execute 325 sec latency 62.000 ms 2024-09-16T08:20:28.018 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:20:27 smithi028 ceph-mon[64115]: pgmap v95: 97 pgs: 15 active+undersized, 24 active+undersized+degraded, 58 active+clean; 343 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 3.6 MiB/s wr, 352 op/s; 175/1626 objects degraded (10.763%) 2024-09-16T08:20:28.054 INFO:teuthology.orchestra.run.smithi028.stdout: 5 191741 10.16 MB/sec execute 326 sec latency 105.107 ms 2024-09-16T08:20:28.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:20:27 smithi144 ceph-mon[49149]: pgmap v95: 97 pgs: 15 active+undersized, 24 active+undersized+degraded, 58 active+clean; 343 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 3.6 MiB/s wr, 352 op/s; 175/1626 objects degraded (10.763%) 2024-09-16T08:20:29.038 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:20:28 smithi028 ceph-mon[64115]: Health check update: Degraded data redundancy: 177/1617 objects degraded (10.946%), 24 pgs degraded (PG_DEGRADED) 2024-09-16T08:20:29.054 INFO:teuthology.orchestra.run.smithi028.stdout: 5 191997 10.14 MB/sec execute 327 sec latency 62.219 ms 2024-09-16T08:20:29.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:20:28 smithi144 ceph-mon[49149]: Health check update: Degraded data redundancy: 177/1617 objects degraded (10.946%), 24 pgs degraded (PG_DEGRADED) 2024-09-16T08:20:30.055 INFO:teuthology.orchestra.run.smithi028.stdout: 5 192201 10.13 MB/sec execute 328 sec latency 52.834 ms 2024-09-16T08:20:30.060 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:20:29 smithi028 ceph-mon[64115]: pgmap v96: 97 pgs: 15 active+undersized, 24 active+undersized+degraded, 58 active+clean; 345 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 8.2 KiB/s rd, 5.1 MiB/s wr, 506 op/s; 177/1617 objects degraded (10.946%) 2024-09-16T08:20:30.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:20:29 smithi144 ceph-mon[49149]: pgmap v96: 97 pgs: 15 active+undersized, 24 active+undersized+degraded, 58 active+clean; 345 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 8.2 KiB/s rd, 5.1 MiB/s wr, 506 op/s; 177/1617 objects degraded (10.946%) 2024-09-16T08:20:31.054 INFO:teuthology.orchestra.run.smithi028.stdout: 5 192514 10.13 MB/sec execute 329 sec latency 81.520 ms 2024-09-16T08:20:31.069 INFO:journalctl@ceph.osd.1.smithi028.stdout:Sep 16 08:20:30 smithi028 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-1[71361]: 2024-09-16T08:20:30.844+0000 7f6be3992640 -1 osd.1 0 read_superblock omap replica is missing. 2024-09-16T08:20:31.573 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:20:31 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:20:31.573 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:20:31 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:20:31.573 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:20:31 smithi028 ceph-mon[64115]: pgmap v97: 97 pgs: 15 active+undersized, 24 active+undersized+degraded, 58 active+clean; 346 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 5.8 KiB/s rd, 3.8 MiB/s wr, 378 op/s; 178/1647 objects degraded (10.808%) 2024-09-16T08:20:31.573 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:20:31 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:20:31.573 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:20:31 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:20:31.833 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:20:31 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:20:31.833 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:20:31 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:20:31.833 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:20:31 smithi144 ceph-mon[49149]: pgmap v97: 97 pgs: 15 active+undersized, 24 active+undersized+degraded, 58 active+clean; 346 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 5.8 KiB/s rd, 3.8 MiB/s wr, 378 op/s; 178/1647 objects degraded (10.808%) 2024-09-16T08:20:31.833 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:20:31 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:20:31.833 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:20:31 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:20:32.054 INFO:teuthology.orchestra.run.smithi028.stdout: 5 192857 10.13 MB/sec execute 330 sec latency 66.122 ms 2024-09-16T08:20:32.328 INFO:journalctl@ceph.osd.1.smithi028.stdout:Sep 16 08:20:32 smithi028 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-1[71361]: 2024-09-16T08:20:32.247+0000 7f6be3992640 -1 osd.1 82 log_to_monitors true 2024-09-16T08:20:32.579 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:20:32 smithi028 ceph-mon[64115]: from='osd.1 [v2:172.21.15.28:6810/1510585938,v1:172.21.15.28:6811/1510585938]' entity='osd.1' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]} : dispatch 2024-09-16T08:20:32.579 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:20:32 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-16T08:20:32.831 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:20:32 smithi144 ceph-mon[49149]: from='osd.1 [v2:172.21.15.28:6810/1510585938,v1:172.21.15.28:6811/1510585938]' entity='osd.1' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]} : dispatch 2024-09-16T08:20:32.832 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:20:32 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-16T08:20:33.056 INFO:teuthology.orchestra.run.smithi028.stdout: 5 193222 10.12 MB/sec execute 331 sec latency 63.406 ms 2024-09-16T08:20:33.085 INFO:journalctl@ceph.osd.1.smithi028.stdout:Sep 16 08:20:32 smithi028 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-1[71361]: 2024-09-16T08:20:32.977+0000 7f6bd7734640 -1 osd.1 82 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-16T08:20:33.602 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:20:33 smithi028 ceph-mon[64115]: from='osd.1 [v2:172.21.15.28:6810/1510585938,v1:172.21.15.28:6811/1510585938]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-09-16T08:20:33.602 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:20:33 smithi028 ceph-mon[64115]: osdmap e85: 8 total, 7 up, 8 in 2024-09-16T08:20:33.602 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:20:33 smithi028 ceph-mon[64115]: from='osd.1 [v2:172.21.15.28:6810/1510585938,v1:172.21.15.28:6811/1510585938]' entity='osd.1' cmd={"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi028", "root=default"]} : dispatch 2024-09-16T08:20:33.602 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:20:33 smithi028 ceph-mon[64115]: pgmap v99: 97 pgs: 15 active+undersized, 24 active+undersized+degraded, 58 active+clean; 355 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.1 KiB/s rd, 5.2 MiB/s wr, 587 op/s; 188/1689 objects degraded (11.131%) 2024-09-16T08:20:33.602 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:20:33 smithi028 ceph-mon[64115]: from='osd.1 [v2:172.21.15.28:6810/1510585938,v1:172.21.15.28:6811/1510585938]' entity='osd.1' 2024-09-16T08:20:34.054 INFO:teuthology.orchestra.run.smithi028.stdout: 5 193548 10.12 MB/sec execute 332 sec latency 157.581 ms 2024-09-16T08:20:34.081 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:20:33 smithi144 ceph-mon[49149]: from='osd.1 [v2:172.21.15.28:6810/1510585938,v1:172.21.15.28:6811/1510585938]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-09-16T08:20:34.082 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:20:33 smithi144 ceph-mon[49149]: osdmap e85: 8 total, 7 up, 8 in 2024-09-16T08:20:34.082 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:20:33 smithi144 ceph-mon[49149]: from='osd.1 [v2:172.21.15.28:6810/1510585938,v1:172.21.15.28:6811/1510585938]' entity='osd.1' cmd={"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi028", "root=default"]} : dispatch 2024-09-16T08:20:34.082 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:20:33 smithi144 ceph-mon[49149]: pgmap v99: 97 pgs: 15 active+undersized, 24 active+undersized+degraded, 58 active+clean; 355 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.1 KiB/s rd, 5.2 MiB/s wr, 587 op/s; 188/1689 objects degraded (11.131%) 2024-09-16T08:20:34.082 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:20:33 smithi144 ceph-mon[49149]: from='osd.1 [v2:172.21.15.28:6810/1510585938,v1:172.21.15.28:6811/1510585938]' entity='osd.1' 2024-09-16T08:20:34.654 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:20:34 smithi028 ceph-mon[64115]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-16T08:20:34.654 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:20:34 smithi028 ceph-mon[64115]: osd.1 [v2:172.21.15.28:6810/1510585938,v1:172.21.15.28:6811/1510585938] boot 2024-09-16T08:20:34.654 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:20:34 smithi028 ceph-mon[64115]: osdmap e86: 8 total, 8 up, 8 in 2024-09-16T08:20:34.654 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:20:34 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "osd metadata", "id": 1} : dispatch 2024-09-16T08:20:35.054 INFO:teuthology.orchestra.run.smithi028.stdout: 5 193654 10.11 MB/sec execute 333 sec latency 1157.609 ms 2024-09-16T08:20:35.081 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:20:34 smithi144 ceph-mon[49149]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-16T08:20:35.082 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:20:34 smithi144 ceph-mon[49149]: osd.1 [v2:172.21.15.28:6810/1510585938,v1:172.21.15.28:6811/1510585938] boot 2024-09-16T08:20:35.082 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:20:34 smithi144 ceph-mon[49149]: osdmap e86: 8 total, 8 up, 8 in 2024-09-16T08:20:35.082 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:20:34 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "osd metadata", "id": 1} : dispatch 2024-09-16T08:20:35.947 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:20:35 smithi028 ceph-mon[64115]: pgmap v101: 97 pgs: 15 active+undersized, 24 active+undersized+degraded, 58 active+clean; 355 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.6 KiB/s rd, 5.4 MiB/s wr, 520 op/s; 182/1719 objects degraded (10.588%) 2024-09-16T08:20:35.947 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:20:35 smithi028 ceph-mon[64115]: Health check update: Degraded data redundancy: 188/1689 objects degraded (11.131%), 24 pgs degraded (PG_DEGRADED) 2024-09-16T08:20:35.947 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:20:35 smithi028 ceph-mon[64115]: osdmap e87: 8 total, 8 up, 8 in 2024-09-16T08:20:36.054 INFO:teuthology.orchestra.run.smithi028.stdout: 5 193657 10.08 MB/sec execute 334 sec latency 2157.641 ms 2024-09-16T08:20:36.081 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:20:35 smithi144 ceph-mon[49149]: pgmap v101: 97 pgs: 15 active+undersized, 24 active+undersized+degraded, 58 active+clean; 355 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.6 KiB/s rd, 5.4 MiB/s wr, 520 op/s; 182/1719 objects degraded (10.588%) 2024-09-16T08:20:36.082 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:20:35 smithi144 ceph-mon[49149]: Health check update: Degraded data redundancy: 188/1689 objects degraded (11.131%), 24 pgs degraded (PG_DEGRADED) 2024-09-16T08:20:36.082 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:20:35 smithi144 ceph-mon[49149]: osdmap e87: 8 total, 8 up, 8 in 2024-09-16T08:20:37.055 INFO:teuthology.orchestra.run.smithi028.stdout: 5 194028 10.06 MB/sec execute 335 sec latency 2169.090 ms 2024-09-16T08:20:37.055 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:20:36 smithi028 ceph-mon[64115]: osdmap e88: 8 total, 8 up, 8 in 2024-09-16T08:20:37.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:20:36 smithi144 ceph-mon[49149]: osdmap e88: 8 total, 8 up, 8 in 2024-09-16T08:20:38.055 INFO:teuthology.orchestra.run.smithi028.stdout: 5 194332 10.05 MB/sec execute 336 sec latency 55.530 ms 2024-09-16T08:20:38.076 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:20:37 smithi028 ceph-mon[64115]: pgmap v104: 97 pgs: 3 peering, 13 active+undersized, 23 active+undersized+degraded, 58 active+clean; 354 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.1 KiB/s rd, 5.2 MiB/s wr, 508 op/s; 171/1689 objects degraded (10.124%) 2024-09-16T08:20:38.076 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:20:37 smithi028 ceph-mon[64115]: osdmap e89: 8 total, 8 up, 8 in 2024-09-16T08:20:38.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:20:37 smithi144 ceph-mon[49149]: pgmap v104: 97 pgs: 3 peering, 13 active+undersized, 23 active+undersized+degraded, 58 active+clean; 354 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.1 KiB/s rd, 5.2 MiB/s wr, 508 op/s; 171/1689 objects degraded (10.124%) 2024-09-16T08:20:38.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:20:37 smithi144 ceph-mon[49149]: osdmap e89: 8 total, 8 up, 8 in 2024-09-16T08:20:39.055 INFO:teuthology.orchestra.run.smithi028.stdout: 5 194647 10.06 MB/sec execute 337 sec latency 147.826 ms 2024-09-16T08:20:39.093 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:20:38 smithi028 ceph-mon[64115]: osdmap e90: 8 total, 8 up, 8 in 2024-09-16T08:20:39.093 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:20:38 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:20:39.093 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:20:38 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:20:39.093 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:20:38 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-16T08:20:39.094 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:20:38 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-16T08:20:39.094 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:20:38 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:20:39.094 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:20:38 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:20:39.094 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:20:38 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-16T08:20:39.094 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:20:38 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "versions"} : dispatch 2024-09-16T08:20:39.094 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:20:38 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "versions"} : dispatch 2024-09-16T08:20:39.094 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:20:38 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "versions"} : dispatch 2024-09-16T08:20:39.095 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:20:38 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16} : dispatch 2024-09-16T08:20:39.095 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:20:38 smithi028 ceph-mon[64115]: osdmap e91: 8 total, 8 up, 8 in 2024-09-16T08:20:39.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:20:38 smithi144 ceph-mon[49149]: osdmap e90: 8 total, 8 up, 8 in 2024-09-16T08:20:39.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:20:38 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:20:39.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:20:38 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:20:39.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:20:38 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-16T08:20:39.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:20:38 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-16T08:20:39.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:20:38 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:20:39.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:20:38 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:20:39.333 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:20:38 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-16T08:20:39.333 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:20:38 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "versions"} : dispatch 2024-09-16T08:20:39.333 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:20:38 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "versions"} : dispatch 2024-09-16T08:20:39.333 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:20:38 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "versions"} : dispatch 2024-09-16T08:20:39.333 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:20:38 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16} : dispatch 2024-09-16T08:20:39.333 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:20:38 smithi144 ceph-mon[49149]: osdmap e91: 8 total, 8 up, 8 in 2024-09-16T08:20:40.055 INFO:teuthology.orchestra.run.smithi028.stdout: 5 194853 10.04 MB/sec execute 338 sec latency 993.576 ms 2024-09-16T08:20:40.177 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:20:40 smithi028 ceph-mon[64115]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-16T08:20:40.177 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:20:40 smithi028 ceph-mon[64115]: Upgrade: unsafe to stop osd(s) at this time (5 PGs are or would become offline) 2024-09-16T08:20:40.177 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:20:40 smithi028 ceph-mon[64115]: pgmap v107: 97 pgs: 1 active+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 5 peering, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 342 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 8.5 KiB/s rd, 3.9 MiB/s wr, 431 op/s; 135/1596 objects degraded (8.459%); 2.0 MiB/s, 54 keys/s, 19 objects/s recovering 2024-09-16T08:20:40.177 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:20:40 smithi028 ceph-mon[64115]: Health check update: Degraded data redundancy: 135/1596 objects degraded (8.459%), 7 pgs degraded (PG_DEGRADED) 2024-09-16T08:20:40.177 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:20:40 smithi028 ceph-mon[64115]: osdmap e92: 8 total, 8 up, 8 in 2024-09-16T08:20:40.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:20:40 smithi144 ceph-mon[49149]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-16T08:20:40.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:20:40 smithi144 ceph-mon[49149]: Upgrade: unsafe to stop osd(s) at this time (5 PGs are or would become offline) 2024-09-16T08:20:40.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:20:40 smithi144 ceph-mon[49149]: pgmap v107: 97 pgs: 1 active+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 5 peering, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 342 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 8.5 KiB/s rd, 3.9 MiB/s wr, 431 op/s; 135/1596 objects degraded (8.459%); 2.0 MiB/s, 54 keys/s, 19 objects/s recovering 2024-09-16T08:20:40.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:20:40 smithi144 ceph-mon[49149]: Health check update: Degraded data redundancy: 135/1596 objects degraded (8.459%), 7 pgs degraded (PG_DEGRADED) 2024-09-16T08:20:40.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:20:40 smithi144 ceph-mon[49149]: osdmap e92: 8 total, 8 up, 8 in 2024-09-16T08:20:41.055 INFO:teuthology.orchestra.run.smithi028.stdout: 5 195338 10.06 MB/sec execute 339 sec latency 892.066 ms 2024-09-16T08:20:41.954 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:20:41 smithi028 ceph-mon[64115]: pgmap v110: 97 pgs: 1 active+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 2 peering, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 342 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 8.5 KiB/s rd, 3.9 MiB/s wr, 427 op/s; 135/1590 objects degraded (8.491%); 2.0 MiB/s, 54 keys/s, 24 objects/s recovering 2024-09-16T08:20:41.954 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:20:41 smithi028 ceph-mon[64115]: osdmap e93: 8 total, 8 up, 8 in 2024-09-16T08:20:42.055 INFO:teuthology.orchestra.run.smithi028.stdout: 5 195859 10.06 MB/sec execute 340 sec latency 36.399 ms 2024-09-16T08:20:42.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:20:41 smithi144 ceph-mon[49149]: pgmap v110: 97 pgs: 1 active+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 2 peering, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 342 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 8.5 KiB/s rd, 3.9 MiB/s wr, 427 op/s; 135/1590 objects degraded (8.491%); 2.0 MiB/s, 54 keys/s, 24 objects/s recovering 2024-09-16T08:20:42.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:20:41 smithi144 ceph-mon[49149]: osdmap e93: 8 total, 8 up, 8 in 2024-09-16T08:20:43.055 INFO:teuthology.orchestra.run.smithi028.stdout: 5 196420 10.07 MB/sec execute 341 sec latency 23.436 ms 2024-09-16T08:20:43.362 INFO:teuthology.orchestra.run.smithi028.stdout:true 2024-09-16T08:20:43.968 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:20:43 smithi028 ceph-mon[64115]: pgmap v112: 97 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 94 active+clean; 346 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 6.3 KiB/s rd, 2.9 MiB/s wr, 221 op/s; 48/1641 objects degraded (2.925%); 3.8 KiB/s, 21 objects/s recovering 2024-09-16T08:20:44.055 INFO:teuthology.orchestra.run.smithi028.stdout: 5 196923 10.09 MB/sec execute 342 sec latency 33.523 ms 2024-09-16T08:20:44.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:20:43 smithi144 ceph-mon[49149]: pgmap v112: 97 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 94 active+clean; 346 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 6.3 KiB/s rd, 2.9 MiB/s wr, 221 op/s; 48/1641 objects degraded (2.925%); 3.8 KiB/s, 21 objects/s recovering 2024-09-16T08:20:44.563 INFO:teuthology.orchestra.run.smithi028.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-16T08:20:44.563 INFO:teuthology.orchestra.run.smithi028.stdout:alertmanager.smithi028 smithi028 *:9093,9094 running (5m) 14s ago 14m 18.2M - 0.27.0 11f11916f8cd a14fcb4cb2b6 2024-09-16T08:20:44.563 INFO:teuthology.orchestra.run.smithi028.stdout:crash.smithi028 smithi028 running (110s) 14s ago 14m 6899k - 19.3.0-4917-g8293d73f 425e316ba8eb 4bb89df2a867 2024-09-16T08:20:44.563 INFO:teuthology.orchestra.run.smithi028.stdout:crash.smithi144 smithi144 running (103s) 99s ago 13m 6895k - 19.3.0-4917-g8293d73f 425e316ba8eb 24a96875d7af 2024-09-16T08:20:44.563 INFO:teuthology.orchestra.run.smithi028.stdout:grafana.smithi028 smithi028 *:3000 running (4m) 14s ago 13m 45.7M - dad864ee21e9 b55ba8b944a1 2024-09-16T08:20:44.564 INFO:teuthology.orchestra.run.smithi028.stdout:mds.foofs.smithi028.srtgev smithi028 running (8m) 14s ago 8m 37.8M - 17.2.0 e1d6a67b021e 250e7a7f754c 2024-09-16T08:20:44.564 INFO:teuthology.orchestra.run.smithi028.stdout:mds.foofs.smithi144.oozgsi smithi144 running (8m) 99s ago 8m 16.6M - 17.2.0 e1d6a67b021e ecf4c2bb0fdc 2024-09-16T08:20:44.564 INFO:teuthology.orchestra.run.smithi028.stdout:mgr.smithi028.azvyzy smithi028 *:8443,9283,8765 running (4m) 14s ago 15m 515M - 19.3.0-4917-g8293d73f 425e316ba8eb e01d4382d03f 2024-09-16T08:20:44.564 INFO:teuthology.orchestra.run.smithi028.stdout:mgr.smithi144.utzszj smithi144 *:8443,9283,8765 running (3m) 99s ago 13m 453M - 19.3.0-4917-g8293d73f 425e316ba8eb 71f7f802bffb 2024-09-16T08:20:44.564 INFO:teuthology.orchestra.run.smithi028.stdout:mon.smithi028 smithi028 running (3m) 14s ago 15m 47.9M 2048M 19.3.0-4917-g8293d73f 425e316ba8eb 14efcedc0088 2024-09-16T08:20:44.564 INFO:teuthology.orchestra.run.smithi028.stdout:mon.smithi144 smithi144 running (2m) 99s ago 13m 38.1M 2048M 19.3.0-4917-g8293d73f 425e316ba8eb c081c5ed44b5 2024-09-16T08:20:44.564 INFO:teuthology.orchestra.run.smithi028.stdout:nfs.foo.0.0.smithi028.dkttgj smithi028 *:2049 running (7m) 14s ago 7m 280M - 4.0 e1d6a67b021e b5dccc45a6ed 2024-09-16T08:20:44.564 INFO:teuthology.orchestra.run.smithi028.stdout:nfs.foo.1.0.smithi144.epipkg smithi144 *:2049 running (7m) 99s ago 7m 87.0M - 4.0 e1d6a67b021e faa89e61f50e 2024-09-16T08:20:44.564 INFO:teuthology.orchestra.run.smithi028.stdout:node-exporter.smithi028 smithi028 *:9100 running (4m) 14s ago 13m 10.0M - 1.7.0 72c9c2088986 9e2b8e8a63a9 2024-09-16T08:20:44.564 INFO:teuthology.orchestra.run.smithi028.stdout:node-exporter.smithi144 smithi144 *:9100 running (4m) 99s ago 13m 10.1M - 1.7.0 72c9c2088986 78834c5e992f 2024-09-16T08:20:44.564 INFO:teuthology.orchestra.run.smithi028.stdout:osd.0 smithi028 running (71s) 14s ago 12m 240M 1937M 19.3.0-4917-g8293d73f 425e316ba8eb 2fc02ba4f14f 2024-09-16T08:20:44.564 INFO:teuthology.orchestra.run.smithi028.stdout:osd.1 smithi028 running (22s) 14s ago 12m 12.2M 1937M 19.3.0-4917-g8293d73f 425e316ba8eb df34bd603d8f 2024-09-16T08:20:44.565 INFO:teuthology.orchestra.run.smithi028.stdout:osd.2 smithi028 running (11m) 14s ago 11m 483M 1937M 17.2.0 e1d6a67b021e 90b27d8375a2 2024-09-16T08:20:44.565 INFO:teuthology.orchestra.run.smithi028.stdout:osd.3 smithi028 running (11m) 14s ago 11m 601M 1937M 17.2.0 e1d6a67b021e dac477d8b04f 2024-09-16T08:20:44.565 INFO:teuthology.orchestra.run.smithi028.stdout:osd.4 smithi144 running (10m) 99s ago 10m 527M 2704M 17.2.0 e1d6a67b021e 86719c60975c 2024-09-16T08:20:44.565 INFO:teuthology.orchestra.run.smithi028.stdout:osd.5 smithi144 running (10m) 99s ago 10m 334M 2704M 17.2.0 e1d6a67b021e 70452c145c60 2024-09-16T08:20:44.565 INFO:teuthology.orchestra.run.smithi028.stdout:osd.6 smithi144 running (10m) 99s ago 10m 533M 2704M 17.2.0 e1d6a67b021e 7f9981be792f 2024-09-16T08:20:44.565 INFO:teuthology.orchestra.run.smithi028.stdout:osd.7 smithi144 running (9m) 99s ago 9m 429M 2704M 17.2.0 e1d6a67b021e dada74ddbe4c 2024-09-16T08:20:44.565 INFO:teuthology.orchestra.run.smithi028.stdout:prometheus.smithi028 smithi028 *:9095 running (3m) 14s ago 13m 50.8M - 2.51.0 1d3b7f56885b 42667809df2b 2024-09-16T08:20:44.988 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:20:44 smithi028 ceph-mon[64115]: from='client.34258 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T08:20:44.988 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:20:44 smithi028 ceph-mon[64115]: from='client.44203 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T08:20:44.988 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:20:44 smithi028 ceph-mon[64115]: Health check update: Degraded data redundancy: 48/1641 objects degraded (2.925%), 3 pgs degraded (PG_DEGRADED) 2024-09-16T08:20:45.055 INFO:teuthology.orchestra.run.smithi028.stdout: 5 197317 10.10 MB/sec execute 343 sec latency 47.378 ms 2024-09-16T08:20:45.285 INFO:teuthology.orchestra.run.smithi028.stdout:{ 2024-09-16T08:20:45.285 INFO:teuthology.orchestra.run.smithi028.stdout: "mon": { 2024-09-16T08:20:45.285 INFO:teuthology.orchestra.run.smithi028.stdout: "ceph version 19.3.0-4917-g8293d73f (8293d73f8690540e843a81caec373f9cc29cf705) squid (dev)": 2 2024-09-16T08:20:45.285 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-09-16T08:20:45.285 INFO:teuthology.orchestra.run.smithi028.stdout: "mgr": { 2024-09-16T08:20:45.285 INFO:teuthology.orchestra.run.smithi028.stdout: "ceph version 19.3.0-4917-g8293d73f (8293d73f8690540e843a81caec373f9cc29cf705) squid (dev)": 2 2024-09-16T08:20:45.285 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-09-16T08:20:45.286 INFO:teuthology.orchestra.run.smithi028.stdout: "osd": { 2024-09-16T08:20:45.286 INFO:teuthology.orchestra.run.smithi028.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 6, 2024-09-16T08:20:45.286 INFO:teuthology.orchestra.run.smithi028.stdout: "ceph version 19.3.0-4917-g8293d73f (8293d73f8690540e843a81caec373f9cc29cf705) squid (dev)": 2 2024-09-16T08:20:45.286 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-09-16T08:20:45.286 INFO:teuthology.orchestra.run.smithi028.stdout: "mds": { 2024-09-16T08:20:45.286 INFO:teuthology.orchestra.run.smithi028.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-16T08:20:45.286 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-09-16T08:20:45.286 INFO:teuthology.orchestra.run.smithi028.stdout: "overall": { 2024-09-16T08:20:45.286 INFO:teuthology.orchestra.run.smithi028.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8, 2024-09-16T08:20:45.286 INFO:teuthology.orchestra.run.smithi028.stdout: "ceph version 19.3.0-4917-g8293d73f (8293d73f8690540e843a81caec373f9cc29cf705) squid (dev)": 6 2024-09-16T08:20:45.286 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-09-16T08:20:45.286 INFO:teuthology.orchestra.run.smithi028.stdout:} 2024-09-16T08:20:45.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:20:44 smithi144 ceph-mon[49149]: from='client.34258 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T08:20:45.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:20:44 smithi144 ceph-mon[49149]: from='client.44203 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T08:20:45.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:20:44 smithi144 ceph-mon[49149]: Health check update: Degraded data redundancy: 48/1641 objects degraded (2.925%), 3 pgs degraded (PG_DEGRADED) 2024-09-16T08:20:45.892 INFO:teuthology.orchestra.run.smithi028.stdout:{ 2024-09-16T08:20:45.893 INFO:teuthology.orchestra.run.smithi028.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:f7ad55f9ccd17ba66aadb19b4d291e67b60ea24e5d8a0c0e92941fd59c81e4ce", 2024-09-16T08:20:45.893 INFO:teuthology.orchestra.run.smithi028.stdout: "in_progress": true, 2024-09-16T08:20:45.893 INFO:teuthology.orchestra.run.smithi028.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-16T08:20:45.893 INFO:teuthology.orchestra.run.smithi028.stdout: "services_complete": [ 2024-09-16T08:20:45.893 INFO:teuthology.orchestra.run.smithi028.stdout: "crash", 2024-09-16T08:20:45.893 INFO:teuthology.orchestra.run.smithi028.stdout: "mon", 2024-09-16T08:20:45.893 INFO:teuthology.orchestra.run.smithi028.stdout: "mgr" 2024-09-16T08:20:45.893 INFO:teuthology.orchestra.run.smithi028.stdout: ], 2024-09-16T08:20:45.893 INFO:teuthology.orchestra.run.smithi028.stdout: "progress": "8/23 daemons upgraded", 2024-09-16T08:20:45.893 INFO:teuthology.orchestra.run.smithi028.stdout: "message": "Currently upgrading osd daemons", 2024-09-16T08:20:45.893 INFO:teuthology.orchestra.run.smithi028.stdout: "is_paused": false 2024-09-16T08:20:45.893 INFO:teuthology.orchestra.run.smithi028.stdout:} 2024-09-16T08:20:45.998 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:20:45 smithi028 ceph-mon[64115]: from='client.44207 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T08:20:45.998 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:20:45 smithi028 ceph-mon[64115]: pgmap v113: 97 pgs: 97 active+clean; 352 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 5.2 KiB/s rd, 4.8 MiB/s wr, 462 op/s; 3.1 KiB/s, 25 objects/s recovering 2024-09-16T08:20:45.998 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:20:45 smithi028 ceph-mon[64115]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 48/1641 objects degraded (2.925%), 3 pgs degraded) 2024-09-16T08:20:45.998 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:20:45 smithi028 ceph-mon[64115]: Cluster is now healthy 2024-09-16T08:20:45.998 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:20:45 smithi028 ceph-mon[64115]: from='client.? 172.21.15.28:0/738482271' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-16T08:20:46.055 INFO:teuthology.orchestra.run.smithi028.stdout: 5 197705 10.09 MB/sec execute 344 sec latency 38.833 ms 2024-09-16T08:20:46.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:20:45 smithi144 ceph-mon[49149]: from='client.44207 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T08:20:46.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:20:45 smithi144 ceph-mon[49149]: pgmap v113: 97 pgs: 97 active+clean; 352 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 5.2 KiB/s rd, 4.8 MiB/s wr, 462 op/s; 3.1 KiB/s, 25 objects/s recovering 2024-09-16T08:20:46.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:20:45 smithi144 ceph-mon[49149]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 48/1641 objects degraded (2.925%), 3 pgs degraded) 2024-09-16T08:20:46.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:20:45 smithi144 ceph-mon[49149]: Cluster is now healthy 2024-09-16T08:20:46.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:20:45 smithi144 ceph-mon[49149]: from='client.? 172.21.15.28:0/738482271' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-16T08:20:46.583 INFO:teuthology.orchestra.run.smithi028.stdout:HEALTH_OK 2024-09-16T08:20:47.016 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:20:46 smithi028 ceph-mon[64115]: from='client.44213 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T08:20:47.016 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:20:46 smithi028 ceph-mon[64115]: from='client.? 172.21.15.28:0/1037097602' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-16T08:20:47.055 INFO:teuthology.orchestra.run.smithi028.stdout: 5 197959 10.07 MB/sec execute 345 sec latency 47.314 ms 2024-09-16T08:20:47.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:20:46 smithi144 ceph-mon[49149]: from='client.44213 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T08:20:47.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:20:46 smithi144 ceph-mon[49149]: from='client.? 172.21.15.28:0/1037097602' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-16T08:20:48.038 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:20:47 smithi028 ceph-mon[64115]: pgmap v114: 97 pgs: 97 active+clean; 349 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 3.9 KiB/s rd, 4.4 MiB/s wr, 434 op/s; 2.3 KiB/s, 19 objects/s recovering 2024-09-16T08:20:48.038 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:20:47 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:20:48.038 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:20:47 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-16T08:20:48.055 INFO:teuthology.orchestra.run.smithi028.stdout: 5 198251 10.08 MB/sec execute 346 sec latency 86.358 ms 2024-09-16T08:20:48.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:20:47 smithi144 ceph-mon[49149]: pgmap v114: 97 pgs: 97 active+clean; 349 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 3.9 KiB/s rd, 4.4 MiB/s wr, 434 op/s; 2.3 KiB/s, 19 objects/s recovering 2024-09-16T08:20:48.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:20:47 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:20:48.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:20:47 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-16T08:20:49.055 INFO:teuthology.orchestra.run.smithi028.stdout: 5 198550 10.07 MB/sec execute 347 sec latency 84.253 ms 2024-09-16T08:20:50.055 INFO:teuthology.orchestra.run.smithi028.stdout: 5 198877 10.07 MB/sec execute 348 sec latency 82.666 ms 2024-09-16T08:20:50.090 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:20:49 smithi028 ceph-mon[64115]: pgmap v115: 97 pgs: 97 active+clean; 346 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 8.7 KiB/s rd, 6.0 MiB/s wr, 564 op/s; 0 B/s, 15 objects/s recovering 2024-09-16T08:20:50.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:20:49 smithi144 ceph-mon[49149]: pgmap v115: 97 pgs: 97 active+clean; 346 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 8.7 KiB/s rd, 6.0 MiB/s wr, 564 op/s; 0 B/s, 15 objects/s recovering 2024-09-16T08:20:51.055 INFO:teuthology.orchestra.run.smithi028.stdout: 5 199313 10.07 MB/sec execute 349 sec latency 32.938 ms 2024-09-16T08:20:52.055 INFO:teuthology.orchestra.run.smithi028.stdout: 5 199708 10.06 MB/sec execute 350 sec latency 44.245 ms 2024-09-16T08:20:52.116 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:20:51 smithi028 ceph-mon[64115]: pgmap v116: 97 pgs: 97 active+clean; 349 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 7.8 KiB/s rd, 5.9 MiB/s wr, 577 op/s; 0 B/s, 13 objects/s recovering 2024-09-16T08:20:52.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:20:51 smithi144 ceph-mon[49149]: pgmap v116: 97 pgs: 97 active+clean; 349 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 7.8 KiB/s rd, 5.9 MiB/s wr, 577 op/s; 0 B/s, 13 objects/s recovering 2024-09-16T08:20:53.055 INFO:teuthology.orchestra.run.smithi028.stdout: 5 200103 10.06 MB/sec execute 351 sec latency 34.000 ms 2024-09-16T08:20:53.380 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:20:53 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:20:53.380 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:20:53 smithi028 ceph-mon[64115]: pgmap v117: 97 pgs: 97 active+clean; 351 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 9.5 KiB/s rd, 5.6 MiB/s wr, 510 op/s; 0 B/s, 11 objects/s recovering 2024-09-16T08:20:53.831 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:20:53 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:20:53.832 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:20:53 smithi144 ceph-mon[49149]: pgmap v117: 97 pgs: 97 active+clean; 351 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 9.5 KiB/s rd, 5.6 MiB/s wr, 510 op/s; 0 B/s, 11 objects/s recovering 2024-09-16T08:20:54.055 INFO:teuthology.orchestra.run.smithi028.stdout: 5 200433 10.08 MB/sec execute 352 sec latency 50.573 ms 2024-09-16T08:20:54.409 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:20:54 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16} : dispatch 2024-09-16T08:20:54.409 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:20:54 smithi028 ceph-mon[64115]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-16T08:20:54.409 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:20:54 smithi028 ceph-mon[64115]: Upgrade: osd.2 is safe to restart 2024-09-16T08:20:54.831 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:20:54 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16} : dispatch 2024-09-16T08:20:54.832 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:20:54 smithi144 ceph-mon[49149]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-16T08:20:54.832 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:20:54 smithi144 ceph-mon[49149]: Upgrade: osd.2 is safe to restart 2024-09-16T08:20:55.055 INFO:teuthology.orchestra.run.smithi028.stdout: 5 200718 10.08 MB/sec execute 353 sec latency 72.919 ms 2024-09-16T08:20:55.456 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:20:55 smithi028 ceph-mon[64115]: pgmap v118: 97 pgs: 97 active+clean; 347 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 6.8 KiB/s rd, 5.9 MiB/s wr, 608 op/s; 0 B/s, 3 objects/s recovering 2024-09-16T08:20:55.831 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:20:55 smithi144 ceph-mon[49149]: pgmap v118: 97 pgs: 97 active+clean; 347 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 6.8 KiB/s rd, 5.9 MiB/s wr, 608 op/s; 0 B/s, 3 objects/s recovering 2024-09-16T08:20:56.055 INFO:teuthology.orchestra.run.smithi028.stdout: 5 201051 10.06 MB/sec execute 354 sec latency 63.790 ms 2024-09-16T08:20:56.717 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:20:56 smithi028 ceph-mon[64115]: Upgrade: Updating osd.2 2024-09-16T08:20:56.717 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:20:56 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:20:56.717 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:20:56 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "auth get", "entity": "osd.2"} : dispatch 2024-09-16T08:20:56.718 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:20:56 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-16T08:20:56.718 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:20:56 smithi028 ceph-mon[64115]: Deploying daemon osd.2 on smithi028 2024-09-16T08:20:57.055 INFO:teuthology.orchestra.run.smithi028.stdout: 5 201350 10.05 MB/sec execute 355 sec latency 107.903 ms 2024-09-16T08:20:57.081 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:20:56 smithi144 ceph-mon[49149]: Upgrade: Updating osd.2 2024-09-16T08:20:57.082 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:20:56 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:20:57.082 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:20:56 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "auth get", "entity": "osd.2"} : dispatch 2024-09-16T08:20:57.082 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:20:56 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-16T08:20:57.082 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:20:56 smithi144 ceph-mon[49149]: Deploying daemon osd.2 on smithi028 2024-09-16T08:20:57.740 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:20:57 smithi028 ceph-mon[64115]: pgmap v119: 97 pgs: 97 active+clean; 347 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 6.8 KiB/s rd, 5.0 MiB/s wr, 484 op/s 2024-09-16T08:20:58.055 INFO:teuthology.orchestra.run.smithi028.stdout: 5 201569 10.04 MB/sec execute 356 sec latency 88.474 ms 2024-09-16T08:20:58.081 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:20:57 smithi144 ceph-mon[49149]: pgmap v119: 97 pgs: 97 active+clean; 347 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 6.8 KiB/s rd, 5.0 MiB/s wr, 484 op/s 2024-09-16T08:20:59.055 INFO:teuthology.orchestra.run.smithi028.stdout: 5 201982 10.04 MB/sec execute 357 sec latency 58.595 ms 2024-09-16T08:21:00.055 INFO:teuthology.orchestra.run.smithi028.stdout: 5 202695 10.08 MB/sec execute 358 sec latency 35.943 ms 2024-09-16T08:21:00.132 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:20:59 smithi028 ceph-mon[64115]: pgmap v120: 97 pgs: 97 active+clean; 349 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 6.2 MiB/s wr, 605 op/s 2024-09-16T08:21:00.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:20:59 smithi144 ceph-mon[49149]: pgmap v120: 97 pgs: 97 active+clean; 349 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 6.2 MiB/s wr, 605 op/s 2024-09-16T08:21:01.055 INFO:teuthology.orchestra.run.smithi028.stdout: 5 203447 10.09 MB/sec execute 359 sec latency 31.405 ms 2024-09-16T08:21:01.928 INFO:journalctl@ceph.osd.2.smithi028.stdout:Sep 16 08:21:01 smithi028 systemd[1]: Stopping Ceph osd.2 for 5622b2fe-7402-11ef-bceb-c7b262605968... 2024-09-16T08:21:01.928 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:21:01 smithi028 ceph-mon[64115]: pgmap v121: 97 pgs: 97 active+clean; 348 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 6.1 KiB/s rd, 4.7 MiB/s wr, 475 op/s 2024-09-16T08:21:02.055 INFO:teuthology.orchestra.run.smithi028.stdout: 5 203770 10.09 MB/sec execute 360 sec latency 46.831 ms 2024-09-16T08:21:02.179 INFO:journalctl@ceph.osd.2.smithi028.stdout:Sep 16 08:21:02 smithi028 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-2[40979]: 2024-09-16T08:21:02.119+0000 7fed24a3a700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.2 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-16T08:21:02.179 INFO:journalctl@ceph.osd.2.smithi028.stdout:Sep 16 08:21:02 smithi028 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-2[40979]: 2024-09-16T08:21:02.119+0000 7fed24a3a700 -1 osd.2 93 *** Got signal Terminated *** 2024-09-16T08:21:02.179 INFO:journalctl@ceph.osd.2.smithi028.stdout:Sep 16 08:21:02 smithi028 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-2[40979]: 2024-09-16T08:21:02.119+0000 7fed24a3a700 -1 osd.2 93 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-16T08:21:02.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:21:01 smithi144 ceph-mon[49149]: pgmap v121: 97 pgs: 97 active+clean; 348 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 6.1 KiB/s rd, 4.7 MiB/s wr, 475 op/s 2024-09-16T08:21:02.947 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:21:02 smithi028 ceph-mon[64115]: osd.2 marked itself down and dead 2024-09-16T08:21:02.947 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:21:02 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-16T08:21:03.055 INFO:teuthology.orchestra.run.smithi028.stdout: 5 204148 10.11 MB/sec execute 361 sec latency 212.689 ms 2024-09-16T08:21:03.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:21:02 smithi144 ceph-mon[49149]: osd.2 marked itself down and dead 2024-09-16T08:21:03.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:21:02 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-16T08:21:03.850 INFO:journalctl@ceph.osd.2.smithi028.stdout:Sep 16 08:21:03 smithi028 podman[73264]: 2024-09-16 08:21:03.571236717 +0000 UTC m=+1.554223512 container died 90b27d8375a26f03ef6dfc69bee34a38a61e13101064e21b2f5b8bc608a74d53 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-2, io.openshift.tags=base centos centos-stream, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=-17.2.0, name=centos-stream, distribution-scope=public, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.component=centos-stream-container, maintainer=Guillaume Abrioux , io.k8s.display-name=CentOS Stream 8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.buildah.version=1.19.8, ceph=True, version=8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, release=754, GIT_CLEAN=True, vcs-type=git, GIT_REPO=https://github.com/ceph/ceph-container.git, build-date=2022-05-03T08:36:31.336870, architecture=x86_64, vendor=Red Hat, Inc., RELEASE=HEAD, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.openshift.expose-services=) 2024-09-16T08:21:04.056 INFO:teuthology.orchestra.run.smithi028.stdout: 5 204180 10.08 MB/sec execute 362 sec latency 1111.815 ms 2024-09-16T08:21:04.107 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:21:03 smithi028 ceph-mon[64115]: pgmap v122: 97 pgs: 97 active+clean; 348 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 6.0 MiB/s wr, 613 op/s 2024-09-16T08:21:04.107 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:21:03 smithi028 ceph-mon[64115]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-16T08:21:04.107 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:21:03 smithi028 ceph-mon[64115]: osdmap e94: 8 total, 7 up, 8 in 2024-09-16T08:21:04.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:21:03 smithi144 ceph-mon[49149]: pgmap v122: 97 pgs: 97 active+clean; 348 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 6.0 MiB/s wr, 613 op/s 2024-09-16T08:21:04.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:21:03 smithi144 ceph-mon[49149]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-16T08:21:04.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:21:03 smithi144 ceph-mon[49149]: osdmap e94: 8 total, 7 up, 8 in 2024-09-16T08:21:04.359 INFO:journalctl@ceph.osd.2.smithi028.stdout:Sep 16 08:21:04 smithi028 podman[73264]: 2024-09-16 08:21:04.333807132 +0000 UTC m=+2.316793920 container cleanup 90b27d8375a26f03ef6dfc69bee34a38a61e13101064e21b2f5b8bc608a74d53 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-2, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=-17.2.0, vendor=Red Hat, Inc., io.openshift.expose-services=, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.openshift.tags=base centos centos-stream, GIT_BRANCH=HEAD, RELEASE=HEAD, distribution-scope=public, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, release=754, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.k8s.display-name=CentOS Stream 8, vcs-type=git, GIT_REPO=https://github.com/ceph/ceph-container.git, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.component=centos-stream-container, GIT_CLEAN=True, build-date=2022-05-03T08:36:31.336870, ceph=True, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.buildah.version=1.19.8, version=8, name=centos-stream, architecture=x86_64) 2024-09-16T08:21:04.359 INFO:journalctl@ceph.osd.2.smithi028.stdout:Sep 16 08:21:04 smithi028 bash[73264]: ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-2 2024-09-16T08:21:04.867 INFO:journalctl@ceph.osd.2.smithi028.stdout:Sep 16 08:21:04 smithi028 podman[73283]: 2024-09-16 08:21:04.685699619 +0000 UTC m=+1.112779424 container remove 90b27d8375a26f03ef6dfc69bee34a38a61e13101064e21b2f5b8bc608a74d53 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-2, vcs-type=git, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., name=centos-stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.k8s.display-name=CentOS Stream 8, architecture=x86_64, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_BRANCH=HEAD, com.redhat.component=centos-stream-container, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, distribution-scope=public, release=754, io.buildah.version=1.19.8, GIT_REPO=https://github.com/ceph/ceph-container.git, io.openshift.tags=base centos centos-stream, RELEASE=HEAD, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, CEPH_POINT_RELEASE=-17.2.0, version=8, maintainer=Guillaume Abrioux , GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.openshift.expose-services=, vendor=Red Hat, Inc., GIT_CLEAN=True, build-date=2022-05-03T08:36:31.336870, ceph=True) 2024-09-16T08:21:05.056 INFO:teuthology.orchestra.run.smithi028.stdout: 5 204613 10.08 MB/sec execute 363 sec latency 30.904 ms 2024-09-16T08:21:05.118 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:21:04 smithi028 ceph-mon[64115]: osdmap e95: 8 total, 7 up, 8 in 2024-09-16T08:21:05.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:21:04 smithi144 ceph-mon[49149]: osdmap e95: 8 total, 7 up, 8 in 2024-09-16T08:21:05.369 INFO:journalctl@ceph.osd.2.smithi028.stdout:Sep 16 08:21:05 smithi028 podman[73376]: 2024-09-16 08:21:05.222927968 +0000 UTC m=+0.021076186 image pull 425e316ba8ebf74f04aa3640f7deb4fa3f293a78d0c8d097cb679eaf86ddca6c quay.ceph.io/ceph-ci/ceph@sha256:f7ad55f9ccd17ba66aadb19b4d291e67b60ea24e5d8a0c0e92941fd59c81e4ce 2024-09-16T08:21:05.369 INFO:journalctl@ceph.osd.2.smithi028.stdout:Sep 16 08:21:05 smithi028 podman[73376]: 2024-09-16 08:21:05.362015928 +0000 UTC m=+0.160164136 container create ba612f458e88454ac500c189ba487adf72c049232d21a107221ce9ff9e10dc6f (image=quay.ceph.io/ceph-ci/ceph@sha256:f7ad55f9ccd17ba66aadb19b4d291e67b60ea24e5d8a0c0e92941fd59c81e4ce, name=ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-2-deactivate, GIT_CLEAN=True, CEPH_POINT_RELEASE=, RELEASE=wip-guits-main-2024-09-13-1248-8293d73, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.license=GPLv2) 2024-09-16T08:21:05.879 INFO:journalctl@ceph.osd.2.smithi028.stdout:Sep 16 08:21:05 smithi028 podman[73376]: 2024-09-16 08:21:05.77499106 +0000 UTC m=+0.573139269 container init ba612f458e88454ac500c189ba487adf72c049232d21a107221ce9ff9e10dc6f (image=quay.ceph.io/ceph-ci/ceph@sha256:f7ad55f9ccd17ba66aadb19b4d291e67b60ea24e5d8a0c0e92941fd59c81e4ce, name=ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-2-deactivate, GIT_CLEAN=True, GIT_BRANCH=HEAD, RELEASE=wip-guits-main-2024-09-13-1248-8293d73, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-16T08:21:05.879 INFO:journalctl@ceph.osd.2.smithi028.stdout:Sep 16 08:21:05 smithi028 podman[73376]: 2024-09-16 08:21:05.787050601 +0000 UTC m=+0.585198812 container start ba612f458e88454ac500c189ba487adf72c049232d21a107221ce9ff9e10dc6f (image=quay.ceph.io/ceph-ci/ceph@sha256:f7ad55f9ccd17ba66aadb19b4d291e67b60ea24e5d8a0c0e92941fd59c81e4ce, name=ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-2-deactivate, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-guits-main-2024-09-13-1248-8293d73, ceph=True, CEPH_POINT_RELEASE=) 2024-09-16T08:21:06.056 INFO:teuthology.orchestra.run.smithi028.stdout: 5 205099 10.08 MB/sec execute 364 sec latency 40.687 ms 2024-09-16T08:21:06.131 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:21:05 smithi028 ceph-mon[64115]: pgmap v125: 97 pgs: 6 stale+active+clean, 91 active+clean; 349 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 6.9 MiB/s wr, 707 op/s 2024-09-16T08:21:06.131 INFO:journalctl@ceph.osd.2.smithi028.stdout:Sep 16 08:21:05 smithi028 podman[73376]: 2024-09-16 08:21:05.959219313 +0000 UTC m=+0.757367533 container attach ba612f458e88454ac500c189ba487adf72c049232d21a107221ce9ff9e10dc6f (image=quay.ceph.io/ceph-ci/ceph@sha256:f7ad55f9ccd17ba66aadb19b4d291e67b60ea24e5d8a0c0e92941fd59c81e4ce, name=ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-2-deactivate, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, GIT_CLEAN=True, RELEASE=wip-guits-main-2024-09-13-1248-8293d73, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD) 2024-09-16T08:21:06.131 INFO:journalctl@ceph.osd.2.smithi028.stdout:Sep 16 08:21:06 smithi028 podman[73376]: 2024-09-16 08:21:06.11932739 +0000 UTC m=+0.917475604 container died ba612f458e88454ac500c189ba487adf72c049232d21a107221ce9ff9e10dc6f (image=quay.ceph.io/ceph-ci/ceph@sha256:f7ad55f9ccd17ba66aadb19b4d291e67b60ea24e5d8a0c0e92941fd59c81e4ce, name=ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-2-deactivate, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_CLEAN=True, RELEASE=wip-guits-main-2024-09-13-1248-8293d73) 2024-09-16T08:21:06.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:21:05 smithi144 ceph-mon[49149]: pgmap v125: 97 pgs: 6 stale+active+clean, 91 active+clean; 349 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 6.9 MiB/s wr, 707 op/s 2024-09-16T08:21:07.056 INFO:teuthology.orchestra.run.smithi028.stdout: 5 205660 10.10 MB/sec execute 365 sec latency 28.095 ms 2024-09-16T08:21:07.145 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:21:06 smithi028 ceph-mon[64115]: Health check failed: Degraded data redundancy: 14/1746 objects degraded (0.802%), 1 pg degraded (PG_DEGRADED) 2024-09-16T08:21:07.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:21:06 smithi144 ceph-mon[49149]: Health check failed: Degraded data redundancy: 14/1746 objects degraded (0.802%), 1 pg degraded (PG_DEGRADED) 2024-09-16T08:21:07.396 INFO:journalctl@ceph.osd.2.smithi028.stdout:Sep 16 08:21:07 smithi028 podman[73397]: 2024-09-16 08:21:07.185948707 +0000 UTC m=+1.047811203 container remove ba612f458e88454ac500c189ba487adf72c049232d21a107221ce9ff9e10dc6f (image=quay.ceph.io/ceph-ci/ceph@sha256:f7ad55f9ccd17ba66aadb19b4d291e67b60ea24e5d8a0c0e92941fd59c81e4ce, name=ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-2-deactivate, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, GIT_CLEAN=True, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, RELEASE=wip-guits-main-2024-09-13-1248-8293d73, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965) 2024-09-16T08:21:07.396 INFO:journalctl@ceph.osd.2.smithi028.stdout:Sep 16 08:21:07 smithi028 systemd[1]: ceph-5622b2fe-7402-11ef-bceb-c7b262605968@osd.2.service: Deactivated successfully. 2024-09-16T08:21:07.396 INFO:journalctl@ceph.osd.2.smithi028.stdout:Sep 16 08:21:07 smithi028 systemd[1]: Stopped Ceph osd.2 for 5622b2fe-7402-11ef-bceb-c7b262605968. 2024-09-16T08:21:07.396 INFO:journalctl@ceph.osd.2.smithi028.stdout:Sep 16 08:21:07 smithi028 systemd[1]: ceph-5622b2fe-7402-11ef-bceb-c7b262605968@osd.2.service: Consumed 2min 43.788s CPU time. 2024-09-16T08:21:07.647 INFO:journalctl@ceph.osd.2.smithi028.stdout:Sep 16 08:21:07 smithi028 systemd[1]: Starting Ceph osd.2 for 5622b2fe-7402-11ef-bceb-c7b262605968... 2024-09-16T08:21:07.899 INFO:journalctl@ceph.osd.2.smithi028.stdout:Sep 16 08:21:07 smithi028 podman[73492]: 2024-09-16 08:21:07.749770818 +0000 UTC m=+0.023529407 image pull 425e316ba8ebf74f04aa3640f7deb4fa3f293a78d0c8d097cb679eaf86ddca6c quay.ceph.io/ceph-ci/ceph@sha256:f7ad55f9ccd17ba66aadb19b4d291e67b60ea24e5d8a0c0e92941fd59c81e4ce 2024-09-16T08:21:07.899 INFO:journalctl@ceph.osd.2.smithi028.stdout:Sep 16 08:21:07 smithi028 podman[73492]: 2024-09-16 08:21:07.862964028 +0000 UTC m=+0.136722598 container create 7c4636c4b3e6ada8d91150e845df93c9472ebd1e913a988953d43996477219f1 (image=quay.ceph.io/ceph-ci/ceph@sha256:f7ad55f9ccd17ba66aadb19b4d291e67b60ea24e5d8a0c0e92941fd59c81e4ce, name=ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-2-activate, ceph=True, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, GIT_CLEAN=True, RELEASE=wip-guits-main-2024-09-13-1248-8293d73, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux ) 2024-09-16T08:21:08.062 INFO:teuthology.orchestra.run.smithi028.stdout: 5 206060 10.11 MB/sec execute 366 sec latency 53.321 ms 2024-09-16T08:21:08.151 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:21:07 smithi028 ceph-mon[64115]: pgmap v126: 97 pgs: 1 active+undersized, 6 stale+active+clean, 1 active+undersized+degraded, 89 active+clean; 349 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 6.6 KiB/s rd, 4.9 MiB/s wr, 469 op/s; 14/1746 objects degraded (0.802%) 2024-09-16T08:21:08.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:21:07 smithi144 ceph-mon[49149]: pgmap v126: 97 pgs: 1 active+undersized, 6 stale+active+clean, 1 active+undersized+degraded, 89 active+clean; 349 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 6.6 KiB/s rd, 4.9 MiB/s wr, 469 op/s; 14/1746 objects degraded (0.802%) 2024-09-16T08:21:08.404 INFO:journalctl@ceph.osd.2.smithi028.stdout:Sep 16 08:21:08 smithi028 podman[73492]: 2024-09-16 08:21:08.250510146 +0000 UTC m=+0.524268708 container init 7c4636c4b3e6ada8d91150e845df93c9472ebd1e913a988953d43996477219f1 (image=quay.ceph.io/ceph-ci/ceph@sha256:f7ad55f9ccd17ba66aadb19b4d291e67b60ea24e5d8a0c0e92941fd59c81e4ce, name=ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-2-activate, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, RELEASE=wip-guits-main-2024-09-13-1248-8293d73, ceph=True, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, CEPH_POINT_RELEASE=) 2024-09-16T08:21:08.404 INFO:journalctl@ceph.osd.2.smithi028.stdout:Sep 16 08:21:08 smithi028 podman[73492]: 2024-09-16 08:21:08.260745979 +0000 UTC m=+0.534504547 container start 7c4636c4b3e6ada8d91150e845df93c9472ebd1e913a988953d43996477219f1 (image=quay.ceph.io/ceph-ci/ceph@sha256:f7ad55f9ccd17ba66aadb19b4d291e67b60ea24e5d8a0c0e92941fd59c81e4ce, name=ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-2-activate, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, RELEASE=wip-guits-main-2024-09-13-1248-8293d73, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965) 2024-09-16T08:21:08.659 INFO:journalctl@ceph.osd.2.smithi028.stdout:Sep 16 08:21:08 smithi028 podman[73492]: 2024-09-16 08:21:08.409688713 +0000 UTC m=+0.683447284 container attach 7c4636c4b3e6ada8d91150e845df93c9472ebd1e913a988953d43996477219f1 (image=quay.ceph.io/ceph-ci/ceph@sha256:f7ad55f9ccd17ba66aadb19b4d291e67b60ea24e5d8a0c0e92941fd59c81e4ce, name=ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-2-activate, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, io.buildah.version=1.37.2, GIT_CLEAN=True, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.build-date=20240731, RELEASE=wip-guits-main-2024-09-13-1248-8293d73, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux ) 2024-09-16T08:21:08.659 INFO:journalctl@ceph.osd.2.smithi028.stdout:Sep 16 08:21:08 smithi028 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-2-activate[73506]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-16T08:21:08.659 INFO:journalctl@ceph.osd.2.smithi028.stdout:Sep 16 08:21:08 smithi028 bash[73492]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-16T08:21:08.659 INFO:journalctl@ceph.osd.2.smithi028.stdout:Sep 16 08:21:08 smithi028 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-2-activate[73506]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-16T08:21:08.659 INFO:journalctl@ceph.osd.2.smithi028.stdout:Sep 16 08:21:08 smithi028 bash[73492]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-16T08:21:09.067 INFO:teuthology.orchestra.run.smithi028.stdout: 5 206497 10.10 MB/sec execute 367 sec latency 77.968 ms 2024-09-16T08:21:10.067 INFO:teuthology.orchestra.run.smithi028.stdout: 5 207004 10.10 MB/sec execute 368 sec latency 34.317 ms 2024-09-16T08:21:10.181 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:21:09 smithi028 ceph-mon[64115]: pgmap v127: 97 pgs: 7 active+undersized, 19 active+undersized+degraded, 71 active+clean; 346 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 9.2 KiB/s rd, 7.7 MiB/s wr, 774 op/s; 189/1767 objects degraded (10.696%) 2024-09-16T08:21:10.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:21:09 smithi144 ceph-mon[49149]: pgmap v127: 97 pgs: 7 active+undersized, 19 active+undersized+degraded, 71 active+clean; 346 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 9.2 KiB/s rd, 7.7 MiB/s wr, 774 op/s; 189/1767 objects degraded (10.696%) 2024-09-16T08:21:11.067 INFO:teuthology.orchestra.run.smithi028.stdout: 5 207524 10.13 MB/sec execute 369 sec latency 27.978 ms 2024-09-16T08:21:11.690 INFO:journalctl@ceph.osd.2.smithi028.stdout:Sep 16 08:21:11 smithi028 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-2-activate[73506]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-09-16T08:21:11.690 INFO:journalctl@ceph.osd.2.smithi028.stdout:Sep 16 08:21:11 smithi028 bash[73492]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-09-16T08:21:11.690 INFO:journalctl@ceph.osd.2.smithi028.stdout:Sep 16 08:21:11 smithi028 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-2-activate[73506]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-2 --no-mon-config --dev /dev/mapper/vg_nvme-lv_2 2024-09-16T08:21:11.690 INFO:journalctl@ceph.osd.2.smithi028.stdout:Sep 16 08:21:11 smithi028 bash[73492]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-2 --no-mon-config --dev /dev/mapper/vg_nvme-lv_2 2024-09-16T08:21:11.942 INFO:journalctl@ceph.osd.2.smithi028.stdout:Sep 16 08:21:11 smithi028 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-2-activate[73506]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_2 2024-09-16T08:21:11.942 INFO:journalctl@ceph.osd.2.smithi028.stdout:Sep 16 08:21:11 smithi028 bash[73492]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_2 2024-09-16T08:21:11.942 INFO:journalctl@ceph.osd.2.smithi028.stdout:Sep 16 08:21:11 smithi028 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-2-activate[73506]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-09-16T08:21:11.942 INFO:journalctl@ceph.osd.2.smithi028.stdout:Sep 16 08:21:11 smithi028 bash[73492]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-09-16T08:21:11.942 INFO:journalctl@ceph.osd.2.smithi028.stdout:Sep 16 08:21:11 smithi028 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-2-activate[73506]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-2/block 2024-09-16T08:21:11.942 INFO:journalctl@ceph.osd.2.smithi028.stdout:Sep 16 08:21:11 smithi028 bash[73492]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-2/block 2024-09-16T08:21:11.942 INFO:journalctl@ceph.osd.2.smithi028.stdout:Sep 16 08:21:11 smithi028 bash[73492]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-09-16T08:21:11.942 INFO:journalctl@ceph.osd.2.smithi028.stdout:Sep 16 08:21:11 smithi028 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-2-activate[73506]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-09-16T08:21:11.942 INFO:journalctl@ceph.osd.2.smithi028.stdout:Sep 16 08:21:11 smithi028 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-2-activate[73506]: --> ceph-volume raw activate successful for osd ID: 2 2024-09-16T08:21:11.943 INFO:journalctl@ceph.osd.2.smithi028.stdout:Sep 16 08:21:11 smithi028 bash[73492]: --> ceph-volume raw activate successful for osd ID: 2 2024-09-16T08:21:11.943 INFO:journalctl@ceph.osd.2.smithi028.stdout:Sep 16 08:21:11 smithi028 podman[73492]: 2024-09-16 08:21:11.912313785 +0000 UTC m=+4.186072347 container died 7c4636c4b3e6ada8d91150e845df93c9472ebd1e913a988953d43996477219f1 (image=quay.ceph.io/ceph-ci/ceph@sha256:f7ad55f9ccd17ba66aadb19b4d291e67b60ea24e5d8a0c0e92941fd59c81e4ce, name=ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-2-activate, org.label-schema.license=GPLv2, RELEASE=wip-guits-main-2024-09-13-1248-8293d73, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.build-date=20240731, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, ceph=True) 2024-09-16T08:21:12.067 INFO:teuthology.orchestra.run.smithi028.stdout: 5 208013 10.13 MB/sec execute 370 sec latency 64.879 ms 2024-09-16T08:21:12.192 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:21:11 smithi028 ceph-mon[64115]: pgmap v128: 97 pgs: 7 active+undersized, 19 active+undersized+degraded, 71 active+clean; 345 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 2.6 KiB/s rd, 5.5 MiB/s wr, 501 op/s; 186/1773 objects degraded (10.491%) 2024-09-16T08:21:12.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:21:11 smithi144 ceph-mon[49149]: pgmap v128: 97 pgs: 7 active+undersized, 19 active+undersized+degraded, 71 active+clean; 345 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 2.6 KiB/s rd, 5.5 MiB/s wr, 501 op/s; 186/1773 objects degraded (10.491%) 2024-09-16T08:21:13.067 INFO:teuthology.orchestra.run.smithi028.stdout: 5 208295 10.13 MB/sec execute 371 sec latency 57.133 ms 2024-09-16T08:21:13.240 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:21:13 smithi028 ceph-mon[64115]: Health check update: Degraded data redundancy: 166/1695 objects degraded (9.794%), 19 pgs degraded (PG_DEGRADED) 2024-09-16T08:21:13.240 INFO:journalctl@ceph.osd.2.smithi028.stdout:Sep 16 08:21:13 smithi028 podman[73670]: 2024-09-16 08:21:13.011393227 +0000 UTC m=+1.091138933 container remove 7c4636c4b3e6ada8d91150e845df93c9472ebd1e913a988953d43996477219f1 (image=quay.ceph.io/ceph-ci/ceph@sha256:f7ad55f9ccd17ba66aadb19b4d291e67b60ea24e5d8a0c0e92941fd59c81e4ce, name=ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-2-activate, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-guits-main-2024-09-13-1248-8293d73, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, ceph=True, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-16T08:21:13.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:21:13 smithi144 ceph-mon[49149]: Health check update: Degraded data redundancy: 166/1695 objects degraded (9.794%), 19 pgs degraded (PG_DEGRADED) 2024-09-16T08:21:13.503 INFO:journalctl@ceph.osd.2.smithi028.stdout:Sep 16 08:21:13 smithi028 podman[73730]: 2024-09-16 08:21:13.145036655 +0000 UTC m=+0.019272316 image pull 425e316ba8ebf74f04aa3640f7deb4fa3f293a78d0c8d097cb679eaf86ddca6c quay.ceph.io/ceph-ci/ceph@sha256:f7ad55f9ccd17ba66aadb19b4d291e67b60ea24e5d8a0c0e92941fd59c81e4ce 2024-09-16T08:21:13.503 INFO:journalctl@ceph.osd.2.smithi028.stdout:Sep 16 08:21:13 smithi028 podman[73730]: 2024-09-16 08:21:13.278323999 +0000 UTC m=+0.152559647 container create c92be6327b129d0890c6956a6187445ba7b46761ace25648ea7e30a60fd67da8 (image=quay.ceph.io/ceph-ci/ceph@sha256:f7ad55f9ccd17ba66aadb19b4d291e67b60ea24e5d8a0c0e92941fd59c81e4ce, name=ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-2, maintainer=Guillaume Abrioux , RELEASE=wip-guits-main-2024-09-13-1248-8293d73, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, ceph=True, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, GIT_CLEAN=True, CEPH_POINT_RELEASE=) 2024-09-16T08:21:13.759 INFO:journalctl@ceph.osd.2.smithi028.stdout:Sep 16 08:21:13 smithi028 podman[73730]: 2024-09-16 08:21:13.615146067 +0000 UTC m=+0.489381727 container init c92be6327b129d0890c6956a6187445ba7b46761ace25648ea7e30a60fd67da8 (image=quay.ceph.io/ceph-ci/ceph@sha256:f7ad55f9ccd17ba66aadb19b4d291e67b60ea24e5d8a0c0e92941fd59c81e4ce, name=ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, RELEASE=wip-guits-main-2024-09-13-1248-8293d73, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_CLEAN=True, ceph=True, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD) 2024-09-16T08:21:13.759 INFO:journalctl@ceph.osd.2.smithi028.stdout:Sep 16 08:21:13 smithi028 podman[73730]: 2024-09-16 08:21:13.625572912 +0000 UTC m=+0.499808561 container start c92be6327b129d0890c6956a6187445ba7b46761ace25648ea7e30a60fd67da8 (image=quay.ceph.io/ceph-ci/ceph@sha256:f7ad55f9ccd17ba66aadb19b4d291e67b60ea24e5d8a0c0e92941fd59c81e4ce, name=ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-2, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, RELEASE=wip-guits-main-2024-09-13-1248-8293d73, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.license=GPLv2, ceph=True, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0) 2024-09-16T08:21:13.759 INFO:journalctl@ceph.osd.2.smithi028.stdout:Sep 16 08:21:13 smithi028 ceph-osd[73748]: -- 172.21.15.28:0/567996897 <== mon.0 v2:172.21.15.28:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x556cbae7a1e0 con 0x556cbae04000 2024-09-16T08:21:13.759 INFO:journalctl@ceph.osd.2.smithi028.stdout:Sep 16 08:21:13 smithi028 bash[73730]: c92be6327b129d0890c6956a6187445ba7b46761ace25648ea7e30a60fd67da8 2024-09-16T08:21:13.760 INFO:journalctl@ceph.osd.2.smithi028.stdout:Sep 16 08:21:13 smithi028 systemd[1]: Started Ceph osd.2 for 5622b2fe-7402-11ef-bceb-c7b262605968. 2024-09-16T08:21:14.067 INFO:teuthology.orchestra.run.smithi028.stdout: 5 208479 10.11 MB/sec execute 372 sec latency 76.682 ms 2024-09-16T08:21:14.266 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:21:14 smithi028 ceph-mon[64115]: pgmap v129: 97 pgs: 7 active+undersized, 19 active+undersized+degraded, 71 active+clean; 339 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 6.6 KiB/s rd, 5.3 MiB/s wr, 507 op/s; 166/1695 objects degraded (9.794%) 2024-09-16T08:21:14.266 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:21:14 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:21:14.266 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:21:14 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:21:14.266 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:21:14 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-16T08:21:14.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:21:14 smithi144 ceph-mon[49149]: pgmap v129: 97 pgs: 7 active+undersized, 19 active+undersized+degraded, 71 active+clean; 339 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 6.6 KiB/s rd, 5.3 MiB/s wr, 507 op/s; 166/1695 objects degraded (9.794%) 2024-09-16T08:21:14.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:21:14 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:21:14.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:21:14 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:21:14.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:21:14 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-16T08:21:15.067 INFO:teuthology.orchestra.run.smithi028.stdout: 5 208731 10.10 MB/sec execute 373 sec latency 58.478 ms 2024-09-16T08:21:15.841 INFO:journalctl@ceph.osd.2.smithi028.stdout:Sep 16 08:21:15 smithi028 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-2[73744]: 2024-09-16T08:21:15.732+0000 7fe356000640 -1 Falling back to public interface 2024-09-16T08:21:16.067 INFO:teuthology.orchestra.run.smithi028.stdout: 5 208967 10.09 MB/sec execute 374 sec latency 59.599 ms 2024-09-16T08:21:16.094 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:21:16 smithi028 ceph-mon[64115]: pgmap v130: 97 pgs: 7 active+undersized, 19 active+undersized+degraded, 71 active+clean; 342 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 5.6 MiB/s wr, 554 op/s; 167/1680 objects degraded (9.940%) 2024-09-16T08:21:16.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:21:16 smithi144 ceph-mon[49149]: pgmap v130: 97 pgs: 7 active+undersized, 19 active+undersized+degraded, 71 active+clean; 342 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 5.6 MiB/s wr, 554 op/s; 167/1680 objects degraded (9.940%) 2024-09-16T08:21:17.067 INFO:teuthology.orchestra.run.smithi028.stdout: 5 209290 10.10 MB/sec execute 375 sec latency 49.292 ms 2024-09-16T08:21:17.184 INFO:teuthology.orchestra.run.smithi028.stdout:true 2024-09-16T08:21:18.067 INFO:teuthology.orchestra.run.smithi028.stdout: 5 209543 10.09 MB/sec execute 376 sec latency 69.548 ms 2024-09-16T08:21:18.133 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:21:18 smithi028 ceph-mon[64115]: pgmap v131: 97 pgs: 7 active+undersized, 19 active+undersized+degraded, 71 active+clean; 342 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 5.5 KiB/s rd, 5.4 MiB/s wr, 521 op/s; 168/1686 objects degraded (9.964%) 2024-09-16T08:21:18.133 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:21:18 smithi028 ceph-mon[64115]: from='client.44223 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T08:21:18.133 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:21:18 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:21:18.133 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:21:18 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-16T08:21:18.133 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:21:18 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:21:18.279 INFO:teuthology.orchestra.run.smithi028.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-16T08:21:18.279 INFO:teuthology.orchestra.run.smithi028.stdout:alertmanager.smithi028 smithi028 *:9093,9094 running (5m) 47s ago 14m 18.2M - 0.27.0 11f11916f8cd a14fcb4cb2b6 2024-09-16T08:21:18.279 INFO:teuthology.orchestra.run.smithi028.stdout:crash.smithi028 smithi028 running (2m) 47s ago 14m 6899k - 19.3.0-4917-g8293d73f 425e316ba8eb 4bb89df2a867 2024-09-16T08:21:18.279 INFO:teuthology.orchestra.run.smithi028.stdout:crash.smithi144 smithi144 running (2m) 2m ago 14m 6895k - 19.3.0-4917-g8293d73f 425e316ba8eb 24a96875d7af 2024-09-16T08:21:18.279 INFO:teuthology.orchestra.run.smithi028.stdout:grafana.smithi028 smithi028 *:3000 running (5m) 47s ago 14m 45.7M - dad864ee21e9 b55ba8b944a1 2024-09-16T08:21:18.279 INFO:teuthology.orchestra.run.smithi028.stdout:mds.foofs.smithi028.srtgev smithi028 running (8m) 47s ago 8m 37.8M - 17.2.0 e1d6a67b021e 250e7a7f754c 2024-09-16T08:21:18.279 INFO:teuthology.orchestra.run.smithi028.stdout:mds.foofs.smithi144.oozgsi smithi144 running (8m) 2m ago 8m 16.6M - 17.2.0 e1d6a67b021e ecf4c2bb0fdc 2024-09-16T08:21:18.280 INFO:teuthology.orchestra.run.smithi028.stdout:mgr.smithi028.azvyzy smithi028 *:8443,9283,8765 running (4m) 47s ago 15m 515M - 19.3.0-4917-g8293d73f 425e316ba8eb e01d4382d03f 2024-09-16T08:21:18.280 INFO:teuthology.orchestra.run.smithi028.stdout:mgr.smithi144.utzszj smithi144 *:8443,9283,8765 running (3m) 2m ago 14m 453M - 19.3.0-4917-g8293d73f 425e316ba8eb 71f7f802bffb 2024-09-16T08:21:18.280 INFO:teuthology.orchestra.run.smithi028.stdout:mon.smithi028 smithi028 running (3m) 47s ago 15m 47.9M 2048M 19.3.0-4917-g8293d73f 425e316ba8eb 14efcedc0088 2024-09-16T08:21:18.280 INFO:teuthology.orchestra.run.smithi028.stdout:mon.smithi144 smithi144 running (3m) 2m ago 13m 38.1M 2048M 19.3.0-4917-g8293d73f 425e316ba8eb c081c5ed44b5 2024-09-16T08:21:18.280 INFO:teuthology.orchestra.run.smithi028.stdout:nfs.foo.0.0.smithi028.dkttgj smithi028 *:2049 running (8m) 47s ago 8m 280M - 4.0 e1d6a67b021e b5dccc45a6ed 2024-09-16T08:21:18.280 INFO:teuthology.orchestra.run.smithi028.stdout:nfs.foo.1.0.smithi144.epipkg smithi144 *:2049 running (8m) 2m ago 8m 87.0M - 4.0 e1d6a67b021e faa89e61f50e 2024-09-16T08:21:18.280 INFO:teuthology.orchestra.run.smithi028.stdout:node-exporter.smithi028 smithi028 *:9100 running (5m) 47s ago 14m 10.0M - 1.7.0 72c9c2088986 9e2b8e8a63a9 2024-09-16T08:21:18.280 INFO:teuthology.orchestra.run.smithi028.stdout:node-exporter.smithi144 smithi144 *:9100 running (4m) 2m ago 13m 10.1M - 1.7.0 72c9c2088986 78834c5e992f 2024-09-16T08:21:18.280 INFO:teuthology.orchestra.run.smithi028.stdout:osd.0 smithi028 running (104s) 47s ago 13m 240M 1937M 19.3.0-4917-g8293d73f 425e316ba8eb 2fc02ba4f14f 2024-09-16T08:21:18.280 INFO:teuthology.orchestra.run.smithi028.stdout:osd.1 smithi028 running (56s) 47s ago 12m 12.2M 1937M 19.3.0-4917-g8293d73f 425e316ba8eb df34bd603d8f 2024-09-16T08:21:18.280 INFO:teuthology.orchestra.run.smithi028.stdout:osd.2 smithi028 starting - - - 1937M 2024-09-16T08:21:18.280 INFO:teuthology.orchestra.run.smithi028.stdout:osd.3 smithi028 running (11m) 47s ago 11m 601M 1937M 17.2.0 e1d6a67b021e dac477d8b04f 2024-09-16T08:21:18.280 INFO:teuthology.orchestra.run.smithi028.stdout:osd.4 smithi144 running (11m) 2m ago 11m 527M 2704M 17.2.0 e1d6a67b021e 86719c60975c 2024-09-16T08:21:18.280 INFO:teuthology.orchestra.run.smithi028.stdout:osd.5 smithi144 running (11m) 2m ago 11m 334M 2704M 17.2.0 e1d6a67b021e 70452c145c60 2024-09-16T08:21:18.281 INFO:teuthology.orchestra.run.smithi028.stdout:osd.6 smithi144 running (10m) 2m ago 10m 533M 2704M 17.2.0 e1d6a67b021e 7f9981be792f 2024-09-16T08:21:18.281 INFO:teuthology.orchestra.run.smithi028.stdout:osd.7 smithi144 running (10m) 2m ago 10m 429M 2704M 17.2.0 e1d6a67b021e dada74ddbe4c 2024-09-16T08:21:18.281 INFO:teuthology.orchestra.run.smithi028.stdout:prometheus.smithi028 smithi028 *:9095 running (4m) 47s ago 13m 50.8M - 2.51.0 1d3b7f56885b 42667809df2b 2024-09-16T08:21:18.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:21:18 smithi144 ceph-mon[49149]: pgmap v131: 97 pgs: 7 active+undersized, 19 active+undersized+degraded, 71 active+clean; 342 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 5.5 KiB/s rd, 5.4 MiB/s wr, 521 op/s; 168/1686 objects degraded (9.964%) 2024-09-16T08:21:18.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:21:18 smithi144 ceph-mon[49149]: from='client.44223 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T08:21:18.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:21:18 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:21:18.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:21:18 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-16T08:21:18.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:21:18 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:21:18.985 INFO:teuthology.orchestra.run.smithi028.stdout:{ 2024-09-16T08:21:18.985 INFO:teuthology.orchestra.run.smithi028.stdout: "mon": { 2024-09-16T08:21:18.985 INFO:teuthology.orchestra.run.smithi028.stdout: "ceph version 19.3.0-4917-g8293d73f (8293d73f8690540e843a81caec373f9cc29cf705) squid (dev)": 2 2024-09-16T08:21:18.985 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-09-16T08:21:18.985 INFO:teuthology.orchestra.run.smithi028.stdout: "mgr": { 2024-09-16T08:21:18.985 INFO:teuthology.orchestra.run.smithi028.stdout: "ceph version 19.3.0-4917-g8293d73f (8293d73f8690540e843a81caec373f9cc29cf705) squid (dev)": 2 2024-09-16T08:21:18.985 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-09-16T08:21:18.986 INFO:teuthology.orchestra.run.smithi028.stdout: "osd": { 2024-09-16T08:21:18.986 INFO:teuthology.orchestra.run.smithi028.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 5, 2024-09-16T08:21:18.986 INFO:teuthology.orchestra.run.smithi028.stdout: "ceph version 19.3.0-4917-g8293d73f (8293d73f8690540e843a81caec373f9cc29cf705) squid (dev)": 2 2024-09-16T08:21:18.986 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-09-16T08:21:18.986 INFO:teuthology.orchestra.run.smithi028.stdout: "mds": { 2024-09-16T08:21:18.986 INFO:teuthology.orchestra.run.smithi028.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-16T08:21:18.986 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-09-16T08:21:18.986 INFO:teuthology.orchestra.run.smithi028.stdout: "overall": { 2024-09-16T08:21:18.986 INFO:teuthology.orchestra.run.smithi028.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 7, 2024-09-16T08:21:18.986 INFO:teuthology.orchestra.run.smithi028.stdout: "ceph version 19.3.0-4917-g8293d73f (8293d73f8690540e843a81caec373f9cc29cf705) squid (dev)": 6 2024-09-16T08:21:18.986 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-09-16T08:21:18.986 INFO:teuthology.orchestra.run.smithi028.stdout:} 2024-09-16T08:21:19.067 INFO:teuthology.orchestra.run.smithi028.stdout: 5 209839 10.08 MB/sec execute 377 sec latency 60.443 ms 2024-09-16T08:21:19.167 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:21:19 smithi028 ceph-mon[64115]: from='client.34284 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T08:21:19.167 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:21:19 smithi028 ceph-mon[64115]: from='client.? 172.21.15.28:0/2809420078' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-16T08:21:19.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:21:19 smithi144 ceph-mon[49149]: from='client.34284 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T08:21:19.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:21:19 smithi144 ceph-mon[49149]: from='client.? 172.21.15.28:0/2809420078' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-16T08:21:19.616 INFO:teuthology.orchestra.run.smithi028.stdout:{ 2024-09-16T08:21:19.616 INFO:teuthology.orchestra.run.smithi028.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:f7ad55f9ccd17ba66aadb19b4d291e67b60ea24e5d8a0c0e92941fd59c81e4ce", 2024-09-16T08:21:19.617 INFO:teuthology.orchestra.run.smithi028.stdout: "in_progress": true, 2024-09-16T08:21:19.617 INFO:teuthology.orchestra.run.smithi028.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-16T08:21:19.617 INFO:teuthology.orchestra.run.smithi028.stdout: "services_complete": [ 2024-09-16T08:21:19.617 INFO:teuthology.orchestra.run.smithi028.stdout: "crash", 2024-09-16T08:21:19.617 INFO:teuthology.orchestra.run.smithi028.stdout: "mon", 2024-09-16T08:21:19.617 INFO:teuthology.orchestra.run.smithi028.stdout: "mgr" 2024-09-16T08:21:19.618 INFO:teuthology.orchestra.run.smithi028.stdout: ], 2024-09-16T08:21:19.618 INFO:teuthology.orchestra.run.smithi028.stdout: "progress": "8/23 daemons upgraded", 2024-09-16T08:21:19.618 INFO:teuthology.orchestra.run.smithi028.stdout: "message": "Currently upgrading osd daemons", 2024-09-16T08:21:19.618 INFO:teuthology.orchestra.run.smithi028.stdout: "is_paused": false 2024-09-16T08:21:19.618 INFO:teuthology.orchestra.run.smithi028.stdout:} 2024-09-16T08:21:20.067 INFO:teuthology.orchestra.run.smithi028.stdout: 5 210022 10.06 MB/sec execute 378 sec latency 70.708 ms 2024-09-16T08:21:20.233 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:21:20 smithi028 ceph-mon[64115]: from='client.34288 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T08:21:20.233 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:21:20 smithi028 ceph-mon[64115]: pgmap v132: 97 pgs: 7 active+undersized, 19 active+undersized+degraded, 71 active+clean; 340 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 8.9 KiB/s rd, 6.4 MiB/s wr, 647 op/s; 161/1590 objects degraded (10.126%) 2024-09-16T08:21:20.233 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:21:20 smithi028 ceph-mon[64115]: Health check update: Degraded data redundancy: 161/1590 objects degraded (10.126%), 19 pgs degraded (PG_DEGRADED) 2024-09-16T08:21:20.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:21:20 smithi144 ceph-mon[49149]: from='client.34288 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T08:21:20.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:21:20 smithi144 ceph-mon[49149]: pgmap v132: 97 pgs: 7 active+undersized, 19 active+undersized+degraded, 71 active+clean; 340 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 8.9 KiB/s rd, 6.4 MiB/s wr, 647 op/s; 161/1590 objects degraded (10.126%) 2024-09-16T08:21:20.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:21:20 smithi144 ceph-mon[49149]: Health check update: Degraded data redundancy: 161/1590 objects degraded (10.126%), 19 pgs degraded (PG_DEGRADED) 2024-09-16T08:21:20.333 INFO:teuthology.orchestra.run.smithi028.stdout:HEALTH_WARN 1 osds down; Degraded data redundancy: 161/1590 objects degraded (10.126%), 19 pgs degraded 2024-09-16T08:21:20.333 INFO:teuthology.orchestra.run.smithi028.stdout:[WRN] OSD_DOWN: 1 osds down 2024-09-16T08:21:20.334 INFO:teuthology.orchestra.run.smithi028.stdout: osd.2 (root=default,host=smithi028) is down 2024-09-16T08:21:20.334 INFO:teuthology.orchestra.run.smithi028.stdout:[WRN] PG_DEGRADED: Degraded data redundancy: 161/1590 objects degraded (10.126%), 19 pgs degraded 2024-09-16T08:21:20.334 INFO:teuthology.orchestra.run.smithi028.stdout: pg 2.1 is active+undersized+degraded, acting [3,0] 2024-09-16T08:21:20.334 INFO:teuthology.orchestra.run.smithi028.stdout: pg 2.3 is active+undersized+degraded, acting [5,7] 2024-09-16T08:21:20.334 INFO:teuthology.orchestra.run.smithi028.stdout: pg 2.7 is active+undersized+degraded, acting [6,7] 2024-09-16T08:21:20.334 INFO:teuthology.orchestra.run.smithi028.stdout: pg 2.c is active+undersized+degraded, acting [5,0] 2024-09-16T08:21:20.334 INFO:teuthology.orchestra.run.smithi028.stdout: pg 2.e is active+undersized+degraded, acting [3,7] 2024-09-16T08:21:20.334 INFO:teuthology.orchestra.run.smithi028.stdout: pg 2.10 is active+undersized+degraded, acting [1,0] 2024-09-16T08:21:20.334 INFO:teuthology.orchestra.run.smithi028.stdout: pg 2.16 is active+undersized+degraded, acting [5,6] 2024-09-16T08:21:20.334 INFO:teuthology.orchestra.run.smithi028.stdout: pg 2.17 is active+undersized+degraded, acting [6,5] 2024-09-16T08:21:20.334 INFO:teuthology.orchestra.run.smithi028.stdout: pg 2.1b is active+undersized+degraded, acting [3,6] 2024-09-16T08:21:20.334 INFO:teuthology.orchestra.run.smithi028.stdout: pg 2.1c is active+undersized+degraded, acting [7,5] 2024-09-16T08:21:20.335 INFO:teuthology.orchestra.run.smithi028.stdout: pg 3.0 is active+undersized+degraded, acting [1,6] 2024-09-16T08:21:20.335 INFO:teuthology.orchestra.run.smithi028.stdout: pg 3.4 is active+undersized+degraded, acting [1,5] 2024-09-16T08:21:20.335 INFO:teuthology.orchestra.run.smithi028.stdout: pg 3.5 is active+undersized+degraded, acting [5,3] 2024-09-16T08:21:20.335 INFO:teuthology.orchestra.run.smithi028.stdout: pg 3.9 is active+undersized+degraded, acting [4,7] 2024-09-16T08:21:20.335 INFO:teuthology.orchestra.run.smithi028.stdout: pg 3.13 is active+undersized+degraded, acting [7,4] 2024-09-16T08:21:20.335 INFO:teuthology.orchestra.run.smithi028.stdout: pg 3.1a is active+undersized+degraded, acting [4,1] 2024-09-16T08:21:20.335 INFO:teuthology.orchestra.run.smithi028.stdout: pg 3.1e is active+undersized+degraded, acting [3,6] 2024-09-16T08:21:20.335 INFO:teuthology.orchestra.run.smithi028.stdout: pg 3.1f is active+undersized+degraded, acting [0,5] 2024-09-16T08:21:20.335 INFO:teuthology.orchestra.run.smithi028.stdout: pg 4.19 is active+undersized+degraded, acting [3,0] 2024-09-16T08:21:21.067 INFO:teuthology.orchestra.run.smithi028.stdout: 5 210319 10.04 MB/sec execute 379 sec latency 49.982 ms 2024-09-16T08:21:21.241 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:21:21 smithi028 ceph-mon[64115]: from='client.44233 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T08:21:21.241 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:21:21 smithi028 ceph-mon[64115]: from='client.? 172.21.15.28:0/3370078171' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-16T08:21:21.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:21:21 smithi144 ceph-mon[49149]: from='client.44233 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T08:21:21.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:21:21 smithi144 ceph-mon[49149]: from='client.? 172.21.15.28:0/3370078171' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-16T08:21:22.012 INFO:journalctl@ceph.osd.2.smithi028.stdout:Sep 16 08:21:21 smithi028 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-2[73744]: 2024-09-16T08:21:21.872+0000 7fe356000640 -1 osd.2 0 read_superblock omap replica is missing. 2024-09-16T08:21:22.067 INFO:teuthology.orchestra.run.smithi028.stdout: 5 210611 10.04 MB/sec execute 380 sec latency 45.277 ms 2024-09-16T08:21:22.266 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:21:22 smithi028 ceph-mon[64115]: pgmap v133: 97 pgs: 7 active+undersized, 19 active+undersized+degraded, 71 active+clean; 340 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 7.2 KiB/s rd, 4.6 MiB/s wr, 445 op/s; 160/1596 objects degraded (10.025%) 2024-09-16T08:21:22.266 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:21:22 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:21:22.266 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:21:22 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:21:22.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:21:22 smithi144 ceph-mon[49149]: pgmap v133: 97 pgs: 7 active+undersized, 19 active+undersized+degraded, 71 active+clean; 340 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 7.2 KiB/s rd, 4.6 MiB/s wr, 445 op/s; 160/1596 objects degraded (10.025%) 2024-09-16T08:21:22.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:21:22 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:21:22.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:21:22 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:21:23.039 INFO:journalctl@ceph.osd.2.smithi028.stdout:Sep 16 08:21:23 smithi028 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-2[73744]: 2024-09-16T08:21:23.030+0000 7fe356000640 -1 osd.2 93 log_to_monitors true 2024-09-16T08:21:23.067 INFO:teuthology.orchestra.run.smithi028.stdout: 5 210874 10.03 MB/sec execute 381 sec latency 56.183 ms 2024-09-16T08:21:24.055 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:21:23 smithi028 ceph-mon[64115]: pgmap v134: 97 pgs: 7 active+undersized, 19 active+undersized+degraded, 71 active+clean; 340 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 7.2 KiB/s rd, 5.1 MiB/s wr, 509 op/s; 155/1602 objects degraded (9.675%) 2024-09-16T08:21:24.055 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:21:23 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:21:24.055 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:21:23 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:21:24.055 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:21:23 smithi028 ceph-mon[64115]: from='osd.2 [v2:172.21.15.28:6818/1160918742,v1:172.21.15.28:6819/1160918742]' entity='osd.2' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]} : dispatch 2024-09-16T08:21:24.067 INFO:teuthology.orchestra.run.smithi028.stdout: 5 211270 10.06 MB/sec execute 382 sec latency 39.587 ms 2024-09-16T08:21:24.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:21:23 smithi144 ceph-mon[49149]: pgmap v134: 97 pgs: 7 active+undersized, 19 active+undersized+degraded, 71 active+clean; 340 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 7.2 KiB/s rd, 5.1 MiB/s wr, 509 op/s; 155/1602 objects degraded (9.675%) 2024-09-16T08:21:24.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:21:23 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:21:24.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:21:23 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:21:24.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:21:23 smithi144 ceph-mon[49149]: from='osd.2 [v2:172.21.15.28:6818/1160918742,v1:172.21.15.28:6819/1160918742]' entity='osd.2' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]} : dispatch 2024-09-16T08:21:24.559 INFO:journalctl@ceph.osd.2.smithi028.stdout:Sep 16 08:21:24 smithi028 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-2[73744]: 2024-09-16T08:21:24.369+0000 7fe349da2640 -1 osd.2 93 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-16T08:21:25.061 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:21:24 smithi028 ceph-mon[64115]: from='osd.2 [v2:172.21.15.28:6818/1160918742,v1:172.21.15.28:6819/1160918742]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-09-16T08:21:25.061 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:21:24 smithi028 ceph-mon[64115]: osdmap e96: 8 total, 7 up, 8 in 2024-09-16T08:21:25.061 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:21:24 smithi028 ceph-mon[64115]: from='osd.2 [v2:172.21.15.28:6818/1160918742,v1:172.21.15.28:6819/1160918742]' entity='osd.2' cmd={"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi028", "root=default"]} : dispatch 2024-09-16T08:21:25.061 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:21:24 smithi028 ceph-mon[64115]: from='osd.2 [v2:172.21.15.28:6818/1160918742,v1:172.21.15.28:6819/1160918742]' entity='osd.2' 2024-09-16T08:21:25.061 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:21:24 smithi028 ceph-mon[64115]: Health check update: Degraded data redundancy: 155/1602 objects degraded (9.675%), 19 pgs degraded (PG_DEGRADED) 2024-09-16T08:21:25.067 INFO:teuthology.orchestra.run.smithi028.stdout: 5 211674 10.05 MB/sec execute 383 sec latency 66.736 ms 2024-09-16T08:21:25.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:21:24 smithi144 ceph-mon[49149]: from='osd.2 [v2:172.21.15.28:6818/1160918742,v1:172.21.15.28:6819/1160918742]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-09-16T08:21:25.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:21:24 smithi144 ceph-mon[49149]: osdmap e96: 8 total, 7 up, 8 in 2024-09-16T08:21:25.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:21:24 smithi144 ceph-mon[49149]: from='osd.2 [v2:172.21.15.28:6818/1160918742,v1:172.21.15.28:6819/1160918742]' entity='osd.2' cmd={"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi028", "root=default"]} : dispatch 2024-09-16T08:21:25.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:21:24 smithi144 ceph-mon[49149]: from='osd.2 [v2:172.21.15.28:6818/1160918742,v1:172.21.15.28:6819/1160918742]' entity='osd.2' 2024-09-16T08:21:25.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:21:24 smithi144 ceph-mon[49149]: Health check update: Degraded data redundancy: 155/1602 objects degraded (9.675%), 19 pgs degraded (PG_DEGRADED) 2024-09-16T08:21:26.067 INFO:teuthology.orchestra.run.smithi028.stdout: 5 211979 10.03 MB/sec execute 384 sec latency 688.111 ms 2024-09-16T08:21:26.072 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:21:25 smithi028 ceph-mon[64115]: pgmap v136: 97 pgs: 7 active+undersized, 19 active+undersized+degraded, 71 active+clean; 344 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.8 KiB/s rd, 4.2 MiB/s wr, 404 op/s; 159/1650 objects degraded (9.636%) 2024-09-16T08:21:26.072 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:21:25 smithi028 ceph-mon[64115]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-16T08:21:26.072 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:21:25 smithi028 ceph-mon[64115]: osd.2 [v2:172.21.15.28:6818/1160918742,v1:172.21.15.28:6819/1160918742] boot 2024-09-16T08:21:26.072 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:21:25 smithi028 ceph-mon[64115]: osdmap e97: 8 total, 8 up, 8 in 2024-09-16T08:21:26.072 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:21:25 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "osd metadata", "id": 2} : dispatch 2024-09-16T08:21:26.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:21:25 smithi144 ceph-mon[49149]: pgmap v136: 97 pgs: 7 active+undersized, 19 active+undersized+degraded, 71 active+clean; 344 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.8 KiB/s rd, 4.2 MiB/s wr, 404 op/s; 159/1650 objects degraded (9.636%) 2024-09-16T08:21:26.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:21:25 smithi144 ceph-mon[49149]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-16T08:21:26.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:21:25 smithi144 ceph-mon[49149]: osd.2 [v2:172.21.15.28:6818/1160918742,v1:172.21.15.28:6819/1160918742] boot 2024-09-16T08:21:26.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:21:25 smithi144 ceph-mon[49149]: osdmap e97: 8 total, 8 up, 8 in 2024-09-16T08:21:26.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:21:25 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "osd metadata", "id": 2} : dispatch 2024-09-16T08:21:27.067 INFO:teuthology.orchestra.run.smithi028.stdout: 5 212102 10.02 MB/sec execute 385 sec latency 1688.165 ms 2024-09-16T08:21:27.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:21:27 smithi144 ceph-mon[49149]: osdmap e98: 8 total, 8 up, 8 in 2024-09-16T08:21:27.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:21:27 smithi144 ceph-mon[49149]: pgmap v139: 97 pgs: 7 active+undersized, 19 active+undersized+degraded, 71 active+clean; 345 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 4.5 KiB/s rd, 3.8 MiB/s wr, 350 op/s; 161/1638 objects degraded (9.829%) 2024-09-16T08:21:27.654 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:21:27 smithi028 ceph-mon[64115]: osdmap e98: 8 total, 8 up, 8 in 2024-09-16T08:21:27.654 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:21:27 smithi028 ceph-mon[64115]: pgmap v139: 97 pgs: 7 active+undersized, 19 active+undersized+degraded, 71 active+clean; 345 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 4.5 KiB/s rd, 3.8 MiB/s wr, 350 op/s; 161/1638 objects degraded (9.829%) 2024-09-16T08:21:28.067 INFO:teuthology.orchestra.run.smithi028.stdout: 5 212168 10.00 MB/sec execute 386 sec latency 2385.335 ms 2024-09-16T08:21:28.410 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:21:28 smithi028 ceph-mon[64115]: osdmap e99: 8 total, 8 up, 8 in 2024-09-16T08:21:28.831 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:21:28 smithi144 ceph-mon[49149]: osdmap e99: 8 total, 8 up, 8 in 2024-09-16T08:21:29.067 INFO:teuthology.orchestra.run.smithi028.stdout: 5 212376 9.99 MB/sec execute 387 sec latency 254.002 ms 2024-09-16T08:21:29.385 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:21:29 smithi028 ceph-mon[64115]: osdmap e100: 8 total, 8 up, 8 in 2024-09-16T08:21:29.385 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:21:29 smithi028 ceph-mon[64115]: pgmap v142: 97 pgs: 2 activating+undersized+degraded+remapped, 1 active+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 3 active+undersized, 5 active+undersized+degraded, 80 active+clean; 343 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 511 B/s rd, 4.0 MiB/s wr, 440 op/s; 224/1602 objects degraded (13.983%); 1.5 MiB/s, 1 objects/s recovering 2024-09-16T08:21:29.385 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:21:29 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:21:29.385 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:21:29 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:21:29.385 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:21:29 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-16T08:21:29.386 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:21:29 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-16T08:21:29.386 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:21:29 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:21:29.386 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:21:29 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:21:29.732 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:21:29 smithi144 ceph-mon[49149]: osdmap e100: 8 total, 8 up, 8 in 2024-09-16T08:21:29.733 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:21:29 smithi144 ceph-mon[49149]: pgmap v142: 97 pgs: 2 activating+undersized+degraded+remapped, 1 active+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 3 active+undersized, 5 active+undersized+degraded, 80 active+clean; 343 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 511 B/s rd, 4.0 MiB/s wr, 440 op/s; 224/1602 objects degraded (13.983%); 1.5 MiB/s, 1 objects/s recovering 2024-09-16T08:21:29.733 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:21:29 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:21:29.733 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:21:29 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:21:29.733 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:21:29 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-16T08:21:29.733 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:21:29 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-16T08:21:29.733 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:21:29 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:21:29.733 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:21:29 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:21:30.067 INFO:teuthology.orchestra.run.smithi028.stdout: 5 212462 9.97 MB/sec execute 388 sec latency 683.406 ms 2024-09-16T08:21:30.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:21:30 smithi028 ceph-mon[64115]: osdmap e101: 8 total, 8 up, 8 in 2024-09-16T08:21:30.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:21:30 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-16T08:21:30.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:21:30 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "versions"} : dispatch 2024-09-16T08:21:30.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:21:30 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "versions"} : dispatch 2024-09-16T08:21:30.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:21:30 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "versions"} : dispatch 2024-09-16T08:21:30.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:21:30 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16} : dispatch 2024-09-16T08:21:30.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:21:30 smithi028 ceph-mon[64115]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-09-16T08:21:30.448 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:21:30 smithi028 ceph-mon[64115]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2024-09-16T08:21:30.448 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:21:30 smithi028 ceph-mon[64115]: Health check update: Degraded data redundancy: 224/1602 objects degraded (13.983%), 13 pgs degraded (PG_DEGRADED) 2024-09-16T08:21:30.832 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:21:30 smithi144 ceph-mon[49149]: osdmap e101: 8 total, 8 up, 8 in 2024-09-16T08:21:30.832 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:21:30 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-16T08:21:30.832 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:21:30 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "versions"} : dispatch 2024-09-16T08:21:30.832 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:21:30 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "versions"} : dispatch 2024-09-16T08:21:30.832 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:21:30 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "versions"} : dispatch 2024-09-16T08:21:30.832 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:21:30 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16} : dispatch 2024-09-16T08:21:30.832 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:21:30 smithi144 ceph-mon[49149]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-09-16T08:21:30.832 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:21:30 smithi144 ceph-mon[49149]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2024-09-16T08:21:30.832 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:21:30 smithi144 ceph-mon[49149]: Health check update: Degraded data redundancy: 224/1602 objects degraded (13.983%), 13 pgs degraded (PG_DEGRADED) 2024-09-16T08:21:31.067 INFO:teuthology.orchestra.run.smithi028.stdout: 5 212616 9.95 MB/sec execute 389 sec latency 984.022 ms 2024-09-16T08:21:31.468 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:21:31 smithi028 ceph-mon[64115]: osdmap e102: 8 total, 8 up, 8 in 2024-09-16T08:21:31.468 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:21:31 smithi028 ceph-mon[64115]: pgmap v145: 97 pgs: 1 peering, 2 activating+undersized+degraded+remapped, 1 active+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 344 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 9.0 KiB/s rd, 3.7 MiB/s wr, 433 op/s; 185/1581 objects degraded (11.701%); 1.5 MiB/s, 7 objects/s recovering 2024-09-16T08:21:31.832 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:21:31 smithi144 ceph-mon[49149]: osdmap e102: 8 total, 8 up, 8 in 2024-09-16T08:21:31.832 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:21:31 smithi144 ceph-mon[49149]: pgmap v145: 97 pgs: 1 peering, 2 activating+undersized+degraded+remapped, 1 active+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 344 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 9.0 KiB/s rd, 3.7 MiB/s wr, 433 op/s; 185/1581 objects degraded (11.701%); 1.5 MiB/s, 7 objects/s recovering 2024-09-16T08:21:32.067 INFO:teuthology.orchestra.run.smithi028.stdout: 5 212862 9.96 MB/sec execute 390 sec latency 695.291 ms 2024-09-16T08:21:32.436 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:21:32 smithi028 ceph-mon[64115]: osdmap e103: 8 total, 8 up, 8 in 2024-09-16T08:21:32.436 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:21:32 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:21:32.436 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:21:32 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-16T08:21:32.832 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:21:32 smithi144 ceph-mon[49149]: osdmap e103: 8 total, 8 up, 8 in 2024-09-16T08:21:32.832 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:21:32 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:21:32.832 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:21:32 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-16T08:21:33.067 INFO:teuthology.orchestra.run.smithi028.stdout: 5 213129 9.94 MB/sec execute 391 sec latency 1043.025 ms 2024-09-16T08:21:33.454 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:21:33 smithi028 ceph-mon[64115]: osdmap e104: 8 total, 8 up, 8 in 2024-09-16T08:21:33.454 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:21:33 smithi028 ceph-mon[64115]: pgmap v148: 97 pgs: 1 peering, 1 active+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 347 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 8.5 KiB/s rd, 3.2 MiB/s wr, 291 op/s; 52/1569 objects degraded (3.314%); 2.3 MiB/s, 39 objects/s recovering 2024-09-16T08:21:33.832 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:21:33 smithi144 ceph-mon[49149]: osdmap e104: 8 total, 8 up, 8 in 2024-09-16T08:21:33.832 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:21:33 smithi144 ceph-mon[49149]: pgmap v148: 97 pgs: 1 peering, 1 active+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 347 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 8.5 KiB/s rd, 3.2 MiB/s wr, 291 op/s; 52/1569 objects degraded (3.314%); 2.3 MiB/s, 39 objects/s recovering 2024-09-16T08:21:34.067 INFO:teuthology.orchestra.run.smithi028.stdout: 5 213502 9.93 MB/sec execute 392 sec latency 43.863 ms 2024-09-16T08:21:35.013 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:21:34 smithi028 ceph-mon[64115]: Health check update: Degraded data redundancy: 52/1569 objects degraded (3.314%), 2 pgs degraded (PG_DEGRADED) 2024-09-16T08:21:35.013 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:21:34 smithi028 ceph-mon[64115]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 52/1569 objects degraded (3.314%), 2 pgs degraded) 2024-09-16T08:21:35.013 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:21:34 smithi028 ceph-mon[64115]: Cluster is now healthy 2024-09-16T08:21:35.067 INFO:teuthology.orchestra.run.smithi028.stdout: 5 213799 9.92 MB/sec execute 393 sec latency 130.479 ms 2024-09-16T08:21:35.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:21:34 smithi144 ceph-mon[49149]: Health check update: Degraded data redundancy: 52/1569 objects degraded (3.314%), 2 pgs degraded (PG_DEGRADED) 2024-09-16T08:21:35.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:21:34 smithi144 ceph-mon[49149]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 52/1569 objects degraded (3.314%), 2 pgs degraded) 2024-09-16T08:21:35.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:21:34 smithi144 ceph-mon[49149]: Cluster is now healthy 2024-09-16T08:21:36.028 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:21:35 smithi028 ceph-mon[64115]: pgmap v149: 97 pgs: 1 peering, 96 active+clean; 347 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 3.6 KiB/s rd, 3.2 MiB/s wr, 294 op/s; 1.6 MiB/s, 37 objects/s recovering 2024-09-16T08:21:36.067 INFO:teuthology.orchestra.run.smithi028.stdout: 5 214200 9.93 MB/sec execute 394 sec latency 53.404 ms 2024-09-16T08:21:36.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:21:35 smithi144 ceph-mon[49149]: pgmap v149: 97 pgs: 1 peering, 96 active+clean; 347 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 3.6 KiB/s rd, 3.2 MiB/s wr, 294 op/s; 1.6 MiB/s, 37 objects/s recovering 2024-09-16T08:21:37.069 INFO:teuthology.orchestra.run.smithi028.stdout: 5 214636 9.93 MB/sec execute 395 sec latency 43.242 ms 2024-09-16T08:21:38.069 INFO:teuthology.orchestra.run.smithi028.stdout: 5 215020 9.95 MB/sec execute 396 sec latency 42.583 ms 2024-09-16T08:21:38.073 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:21:37 smithi028 ceph-mon[64115]: pgmap v150: 97 pgs: 97 active+clean; 346 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 3.1 KiB/s rd, 2.7 MiB/s wr, 256 op/s; 1.4 MiB/s, 28 objects/s recovering 2024-09-16T08:21:38.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:21:37 smithi144 ceph-mon[49149]: pgmap v150: 97 pgs: 97 active+clean; 346 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 3.1 KiB/s rd, 2.7 MiB/s wr, 256 op/s; 1.4 MiB/s, 28 objects/s recovering 2024-09-16T08:21:39.069 INFO:teuthology.orchestra.run.smithi028.stdout: 5 215364 9.95 MB/sec execute 397 sec latency 46.732 ms 2024-09-16T08:21:40.069 INFO:teuthology.orchestra.run.smithi028.stdout: 5 215721 9.94 MB/sec execute 398 sec latency 35.802 ms 2024-09-16T08:21:40.133 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:21:39 smithi028 ceph-mon[64115]: pgmap v151: 97 pgs: 97 active+clean; 352 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 2.7 KiB/s rd, 4.8 MiB/s wr, 490 op/s; 1.1 MiB/s, 23 objects/s recovering 2024-09-16T08:21:40.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:21:39 smithi144 ceph-mon[49149]: pgmap v151: 97 pgs: 97 active+clean; 352 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 2.7 KiB/s rd, 4.8 MiB/s wr, 490 op/s; 1.1 MiB/s, 23 objects/s recovering 2024-09-16T08:21:41.069 INFO:teuthology.orchestra.run.smithi028.stdout: 5 216059 9.94 MB/sec execute 399 sec latency 70.025 ms 2024-09-16T08:21:42.069 INFO:teuthology.orchestra.run.smithi028.stdout: 5 216432 9.94 MB/sec execute 400 sec latency 59.790 ms 2024-09-16T08:21:42.154 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:21:41 smithi028 ceph-mon[64115]: pgmap v152: 97 pgs: 97 active+clean; 349 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 5.8 KiB/s rd, 4.6 MiB/s wr, 437 op/s; 972 KiB/s, 19 objects/s recovering 2024-09-16T08:21:42.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:21:41 smithi144 ceph-mon[49149]: pgmap v152: 97 pgs: 97 active+clean; 349 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 5.8 KiB/s rd, 4.6 MiB/s wr, 437 op/s; 972 KiB/s, 19 objects/s recovering 2024-09-16T08:21:43.069 INFO:teuthology.orchestra.run.smithi028.stdout: 5 216895 9.94 MB/sec execute 401 sec latency 48.476 ms 2024-09-16T08:21:43.415 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:21:43 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:21:43.415 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:21:43 smithi028 ceph-mon[64115]: pgmap v153: 97 pgs: 97 active+clean; 352 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 5.2 KiB/s rd, 5.3 MiB/s wr, 516 op/s; 879 KiB/s, 17 objects/s recovering 2024-09-16T08:21:43.831 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:21:43 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:21:43.832 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:21:43 smithi144 ceph-mon[49149]: pgmap v153: 97 pgs: 97 active+clean; 352 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 5.2 KiB/s rd, 5.3 MiB/s wr, 516 op/s; 879 KiB/s, 17 objects/s recovering 2024-09-16T08:21:44.069 INFO:teuthology.orchestra.run.smithi028.stdout: 5 217322 9.94 MB/sec execute 402 sec latency 34.028 ms 2024-09-16T08:21:44.937 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:21:44 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16} : dispatch 2024-09-16T08:21:45.069 INFO:teuthology.orchestra.run.smithi028.stdout: 5 217699 9.94 MB/sec execute 403 sec latency 86.498 ms 2024-09-16T08:21:45.081 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:21:44 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16} : dispatch 2024-09-16T08:21:45.968 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:21:45 smithi028 ceph-mon[64115]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-09-16T08:21:45.968 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:21:45 smithi028 ceph-mon[64115]: Upgrade: osd.3 is safe to restart 2024-09-16T08:21:45.968 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:21:45 smithi028 ceph-mon[64115]: pgmap v154: 97 pgs: 97 active+clean; 352 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 8.1 KiB/s rd, 4.7 MiB/s wr, 477 op/s; 0 B/s, 4 objects/s recovering 2024-09-16T08:21:46.069 INFO:teuthology.orchestra.run.smithi028.stdout: 5 217996 9.93 MB/sec execute 404 sec latency 69.061 ms 2024-09-16T08:21:46.081 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:21:45 smithi144 ceph-mon[49149]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-09-16T08:21:46.082 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:21:45 smithi144 ceph-mon[49149]: Upgrade: osd.3 is safe to restart 2024-09-16T08:21:46.082 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:21:45 smithi144 ceph-mon[49149]: pgmap v154: 97 pgs: 97 active+clean; 352 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 8.1 KiB/s rd, 4.7 MiB/s wr, 477 op/s; 0 B/s, 4 objects/s recovering 2024-09-16T08:21:47.069 INFO:teuthology.orchestra.run.smithi028.stdout: 5 218356 9.94 MB/sec execute 405 sec latency 62.736 ms 2024-09-16T08:21:48.017 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:21:48 smithi028 ceph-mon[64115]: pgmap v155: 97 pgs: 97 active+clean; 351 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 6.4 KiB/s rd, 4.7 MiB/s wr, 456 op/s 2024-09-16T08:21:48.017 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:21:48 smithi028 ceph-mon[64115]: Upgrade: Updating osd.3 2024-09-16T08:21:48.017 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:21:48 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:21:48.017 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:21:48 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "auth get", "entity": "osd.3"} : dispatch 2024-09-16T08:21:48.017 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:21:48 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-16T08:21:48.018 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:21:48 smithi028 ceph-mon[64115]: Deploying daemon osd.3 on smithi028 2024-09-16T08:21:48.018 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:21:48 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:21:48.018 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:21:48 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-16T08:21:48.069 INFO:teuthology.orchestra.run.smithi028.stdout: 5 218629 9.94 MB/sec execute 406 sec latency 70.461 ms 2024-09-16T08:21:48.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:21:48 smithi144 ceph-mon[49149]: pgmap v155: 97 pgs: 97 active+clean; 351 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 6.4 KiB/s rd, 4.7 MiB/s wr, 456 op/s 2024-09-16T08:21:48.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:21:48 smithi144 ceph-mon[49149]: Upgrade: Updating osd.3 2024-09-16T08:21:48.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:21:48 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:21:48.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:21:48 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "auth get", "entity": "osd.3"} : dispatch 2024-09-16T08:21:48.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:21:48 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-16T08:21:48.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:21:48 smithi144 ceph-mon[49149]: Deploying daemon osd.3 on smithi028 2024-09-16T08:21:48.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:21:48 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:21:48.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:21:48 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-16T08:21:49.069 INFO:teuthology.orchestra.run.smithi028.stdout: 5 218893 9.94 MB/sec execute 407 sec latency 58.260 ms 2024-09-16T08:21:50.070 INFO:teuthology.orchestra.run.smithi028.stdout: 5 219283 9.93 MB/sec execute 408 sec latency 86.291 ms 2024-09-16T08:21:50.079 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:21:50 smithi028 ceph-mon[64115]: pgmap v156: 97 pgs: 97 active+clean; 347 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 6.6 KiB/s rd, 6.3 MiB/s wr, 644 op/s 2024-09-16T08:21:50.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:21:50 smithi144 ceph-mon[49149]: pgmap v156: 97 pgs: 97 active+clean; 347 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 6.6 KiB/s rd, 6.3 MiB/s wr, 644 op/s 2024-09-16T08:21:50.856 INFO:teuthology.orchestra.run.smithi028.stdout:true 2024-09-16T08:21:51.069 INFO:teuthology.orchestra.run.smithi028.stdout: 5 219506 9.92 MB/sec execute 409 sec latency 46.719 ms 2024-09-16T08:21:51.896 INFO:teuthology.orchestra.run.smithi028.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-16T08:21:51.896 INFO:teuthology.orchestra.run.smithi028.stdout:alertmanager.smithi028 smithi028 *:9093,9094 running (6m) 29s ago 15m 18.4M - 0.27.0 11f11916f8cd a14fcb4cb2b6 2024-09-16T08:21:51.896 INFO:teuthology.orchestra.run.smithi028.stdout:crash.smithi028 smithi028 running (2m) 29s ago 15m 6899k - 19.3.0-4917-g8293d73f 425e316ba8eb 4bb89df2a867 2024-09-16T08:21:51.896 INFO:teuthology.orchestra.run.smithi028.stdout:crash.smithi144 smithi144 running (2m) 2m ago 14m 6895k - 19.3.0-4917-g8293d73f 425e316ba8eb 24a96875d7af 2024-09-16T08:21:51.896 INFO:teuthology.orchestra.run.smithi028.stdout:grafana.smithi028 smithi028 *:3000 running (6m) 29s ago 14m 45.8M - dad864ee21e9 b55ba8b944a1 2024-09-16T08:21:51.896 INFO:teuthology.orchestra.run.smithi028.stdout:mds.foofs.smithi028.srtgev smithi028 running (9m) 29s ago 9m 37.8M - 17.2.0 e1d6a67b021e 250e7a7f754c 2024-09-16T08:21:51.896 INFO:teuthology.orchestra.run.smithi028.stdout:mds.foofs.smithi144.oozgsi smithi144 running (9m) 2m ago 9m 16.6M - 17.2.0 e1d6a67b021e ecf4c2bb0fdc 2024-09-16T08:21:51.896 INFO:teuthology.orchestra.run.smithi028.stdout:mgr.smithi028.azvyzy smithi028 *:8443,9283,8765 running (5m) 29s ago 16m 517M - 19.3.0-4917-g8293d73f 425e316ba8eb e01d4382d03f 2024-09-16T08:21:51.896 INFO:teuthology.orchestra.run.smithi028.stdout:mgr.smithi144.utzszj smithi144 *:8443,9283,8765 running (4m) 2m ago 14m 453M - 19.3.0-4917-g8293d73f 425e316ba8eb 71f7f802bffb 2024-09-16T08:21:51.896 INFO:teuthology.orchestra.run.smithi028.stdout:mon.smithi028 smithi028 running (4m) 29s ago 16m 48.3M 2048M 19.3.0-4917-g8293d73f 425e316ba8eb 14efcedc0088 2024-09-16T08:21:51.897 INFO:teuthology.orchestra.run.smithi028.stdout:mon.smithi144 smithi144 running (3m) 2m ago 14m 38.1M 2048M 19.3.0-4917-g8293d73f 425e316ba8eb c081c5ed44b5 2024-09-16T08:21:51.897 INFO:teuthology.orchestra.run.smithi028.stdout:nfs.foo.0.0.smithi028.dkttgj smithi028 *:2049 running (9m) 29s ago 9m 285M - 4.0 e1d6a67b021e b5dccc45a6ed 2024-09-16T08:21:51.897 INFO:teuthology.orchestra.run.smithi028.stdout:nfs.foo.1.0.smithi144.epipkg smithi144 *:2049 running (8m) 2m ago 8m 87.0M - 4.0 e1d6a67b021e faa89e61f50e 2024-09-16T08:21:51.897 INFO:teuthology.orchestra.run.smithi028.stdout:node-exporter.smithi028 smithi028 *:9100 running (5m) 29s ago 14m 9617k - 1.7.0 72c9c2088986 9e2b8e8a63a9 2024-09-16T08:21:51.897 INFO:teuthology.orchestra.run.smithi028.stdout:node-exporter.smithi144 smithi144 *:9100 running (5m) 2m ago 14m 10.1M - 1.7.0 72c9c2088986 78834c5e992f 2024-09-16T08:21:51.897 INFO:teuthology.orchestra.run.smithi028.stdout:osd.0 smithi028 running (2m) 29s ago 13m 297M 1937M 19.3.0-4917-g8293d73f 425e316ba8eb 2fc02ba4f14f 2024-09-16T08:21:51.897 INFO:teuthology.orchestra.run.smithi028.stdout:osd.1 smithi028 running (89s) 29s ago 13m 229M 1937M 19.3.0-4917-g8293d73f 425e316ba8eb df34bd603d8f 2024-09-16T08:21:51.897 INFO:teuthology.orchestra.run.smithi028.stdout:osd.2 smithi028 running (38s) 29s ago 12m 12.0M 1937M 19.3.0-4917-g8293d73f 425e316ba8eb c92be6327b12 2024-09-16T08:21:51.897 INFO:teuthology.orchestra.run.smithi028.stdout:osd.3 smithi028 running (12m) 29s ago 12m 381M 1937M 17.2.0 e1d6a67b021e dac477d8b04f 2024-09-16T08:21:51.897 INFO:teuthology.orchestra.run.smithi028.stdout:osd.4 smithi144 running (12m) 2m ago 12m 527M 2704M 17.2.0 e1d6a67b021e 86719c60975c 2024-09-16T08:21:51.897 INFO:teuthology.orchestra.run.smithi028.stdout:osd.5 smithi144 running (11m) 2m ago 11m 334M 2704M 17.2.0 e1d6a67b021e 70452c145c60 2024-09-16T08:21:51.897 INFO:teuthology.orchestra.run.smithi028.stdout:osd.6 smithi144 running (11m) 2m ago 11m 533M 2704M 17.2.0 e1d6a67b021e 7f9981be792f 2024-09-16T08:21:51.897 INFO:teuthology.orchestra.run.smithi028.stdout:osd.7 smithi144 running (10m) 2m ago 10m 429M 2704M 17.2.0 e1d6a67b021e dada74ddbe4c 2024-09-16T08:21:51.897 INFO:teuthology.orchestra.run.smithi028.stdout:prometheus.smithi028 smithi028 *:9095 running (4m) 29s ago 14m 51.0M - 2.51.0 1d3b7f56885b 42667809df2b 2024-09-16T08:21:52.070 INFO:teuthology.orchestra.run.smithi028.stdout: 5 219735 9.91 MB/sec execute 410 sec latency 54.668 ms 2024-09-16T08:21:52.107 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:21:52 smithi028 ceph-mon[64115]: from='client.34306 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T08:21:52.107 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:21:52 smithi028 ceph-mon[64115]: pgmap v157: 97 pgs: 97 active+clean; 348 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 9.2 KiB/s rd, 4.8 MiB/s wr, 467 op/s 2024-09-16T08:21:52.108 INFO:journalctl@ceph.osd.3.smithi028.stdout:Sep 16 08:21:52 smithi028 systemd[1]: Stopping Ceph osd.3 for 5622b2fe-7402-11ef-bceb-c7b262605968... 2024-09-16T08:21:52.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:21:52 smithi144 ceph-mon[49149]: from='client.34306 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T08:21:52.333 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:21:52 smithi144 ceph-mon[49149]: pgmap v157: 97 pgs: 97 active+clean; 348 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 9.2 KiB/s rd, 4.8 MiB/s wr, 467 op/s 2024-09-16T08:21:52.562 INFO:teuthology.orchestra.run.smithi028.stdout:{ 2024-09-16T08:21:52.562 INFO:teuthology.orchestra.run.smithi028.stdout: "mon": { 2024-09-16T08:21:52.562 INFO:teuthology.orchestra.run.smithi028.stdout: "ceph version 19.3.0-4917-g8293d73f (8293d73f8690540e843a81caec373f9cc29cf705) squid (dev)": 2 2024-09-16T08:21:52.562 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-09-16T08:21:52.562 INFO:teuthology.orchestra.run.smithi028.stdout: "mgr": { 2024-09-16T08:21:52.562 INFO:teuthology.orchestra.run.smithi028.stdout: "ceph version 19.3.0-4917-g8293d73f (8293d73f8690540e843a81caec373f9cc29cf705) squid (dev)": 2 2024-09-16T08:21:52.563 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-09-16T08:21:52.563 INFO:teuthology.orchestra.run.smithi028.stdout: "osd": { 2024-09-16T08:21:52.563 INFO:teuthology.orchestra.run.smithi028.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 5, 2024-09-16T08:21:52.563 INFO:teuthology.orchestra.run.smithi028.stdout: "ceph version 19.3.0-4917-g8293d73f (8293d73f8690540e843a81caec373f9cc29cf705) squid (dev)": 3 2024-09-16T08:21:52.563 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-09-16T08:21:52.563 INFO:teuthology.orchestra.run.smithi028.stdout: "mds": { 2024-09-16T08:21:52.563 INFO:teuthology.orchestra.run.smithi028.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-16T08:21:52.563 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-09-16T08:21:52.563 INFO:teuthology.orchestra.run.smithi028.stdout: "overall": { 2024-09-16T08:21:52.563 INFO:teuthology.orchestra.run.smithi028.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 7, 2024-09-16T08:21:52.563 INFO:teuthology.orchestra.run.smithi028.stdout: "ceph version 19.3.0-4917-g8293d73f (8293d73f8690540e843a81caec373f9cc29cf705) squid (dev)": 7 2024-09-16T08:21:52.563 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-09-16T08:21:52.563 INFO:teuthology.orchestra.run.smithi028.stdout:} 2024-09-16T08:21:52.613 INFO:journalctl@ceph.osd.3.smithi028.stdout:Sep 16 08:21:52 smithi028 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-3[44733]: 2024-09-16T08:21:52.472+0000 7ff56ffb6700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.3 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-16T08:21:52.614 INFO:journalctl@ceph.osd.3.smithi028.stdout:Sep 16 08:21:52 smithi028 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-3[44733]: 2024-09-16T08:21:52.472+0000 7ff56ffb6700 -1 osd.3 104 *** Got signal Terminated *** 2024-09-16T08:21:52.614 INFO:journalctl@ceph.osd.3.smithi028.stdout:Sep 16 08:21:52 smithi028 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-3[44733]: 2024-09-16T08:21:52.472+0000 7ff56ffb6700 -1 osd.3 104 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-16T08:21:53.070 INFO:teuthology.orchestra.run.smithi028.stdout: 5 220118 9.92 MB/sec execute 411 sec latency 44.221 ms 2024-09-16T08:21:53.113 INFO:teuthology.orchestra.run.smithi028.stdout:{ 2024-09-16T08:21:53.113 INFO:teuthology.orchestra.run.smithi028.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:f7ad55f9ccd17ba66aadb19b4d291e67b60ea24e5d8a0c0e92941fd59c81e4ce", 2024-09-16T08:21:53.113 INFO:teuthology.orchestra.run.smithi028.stdout: "in_progress": true, 2024-09-16T08:21:53.113 INFO:teuthology.orchestra.run.smithi028.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-16T08:21:53.113 INFO:teuthology.orchestra.run.smithi028.stdout: "services_complete": [ 2024-09-16T08:21:53.113 INFO:teuthology.orchestra.run.smithi028.stdout: "crash", 2024-09-16T08:21:53.113 INFO:teuthology.orchestra.run.smithi028.stdout: "mon", 2024-09-16T08:21:53.113 INFO:teuthology.orchestra.run.smithi028.stdout: "mgr" 2024-09-16T08:21:53.114 INFO:teuthology.orchestra.run.smithi028.stdout: ], 2024-09-16T08:21:53.114 INFO:teuthology.orchestra.run.smithi028.stdout: "progress": "9/23 daemons upgraded", 2024-09-16T08:21:53.114 INFO:teuthology.orchestra.run.smithi028.stdout: "message": "Currently upgrading osd daemons", 2024-09-16T08:21:53.114 INFO:teuthology.orchestra.run.smithi028.stdout: "is_paused": false 2024-09-16T08:21:53.114 INFO:teuthology.orchestra.run.smithi028.stdout:} 2024-09-16T08:21:53.122 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:21:53 smithi028 ceph-mon[64115]: from='client.34310 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T08:21:53.122 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:21:53 smithi028 ceph-mon[64115]: from='client.34314 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T08:21:53.122 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:21:53 smithi028 ceph-mon[64115]: osd.3 marked itself down and dead 2024-09-16T08:21:53.122 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:21:53 smithi028 ceph-mon[64115]: from='client.? 172.21.15.28:0/1721695417' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-16T08:21:53.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:21:53 smithi144 ceph-mon[49149]: from='client.34310 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T08:21:53.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:21:53 smithi144 ceph-mon[49149]: from='client.34314 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T08:21:53.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:21:53 smithi144 ceph-mon[49149]: osd.3 marked itself down and dead 2024-09-16T08:21:53.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:21:53 smithi144 ceph-mon[49149]: from='client.? 172.21.15.28:0/1721695417' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-16T08:21:53.710 INFO:teuthology.orchestra.run.smithi028.stdout:HEALTH_WARN 1 osds down 2024-09-16T08:21:53.710 INFO:teuthology.orchestra.run.smithi028.stdout:[WRN] OSD_DOWN: 1 osds down 2024-09-16T08:21:53.710 INFO:teuthology.orchestra.run.smithi028.stdout: osd.3 (root=default,host=smithi028) is down 2024-09-16T08:21:53.947 INFO:journalctl@ceph.osd.3.smithi028.stdout:Sep 16 08:21:53 smithi028 podman[75716]: 2024-09-16 08:21:53.779990087 +0000 UTC m=+1.466653503 container died dac477d8b04f805426c3b38a4cf7091a8d57debdc1edaa558665de349d16cbe1 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-3, RELEASE=HEAD, maintainer=Guillaume Abrioux , url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, architecture=x86_64, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vendor=Red Hat, Inc., CEPH_POINT_RELEASE=-17.2.0, io.buildah.version=1.19.8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_CLEAN=True, version=8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, name=centos-stream, release=754, com.redhat.component=centos-stream-container, io.openshift.tags=base centos centos-stream, io.openshift.expose-services=, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_REPO=https://github.com/ceph/ceph-container.git, io.k8s.display-name=CentOS Stream 8, build-date=2022-05-03T08:36:31.336870, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, distribution-scope=public, vcs-type=git, ceph=True, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_BRANCH=HEAD) 2024-09-16T08:21:54.070 INFO:teuthology.orchestra.run.smithi028.stdout: 5 220263 9.90 MB/sec execute 412 sec latency 998.332 ms 2024-09-16T08:21:54.279 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:21:54 smithi028 ceph-mon[64115]: pgmap v158: 97 pgs: 97 active+clean; 351 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 6.5 KiB/s rd, 5.3 MiB/s wr, 527 op/s 2024-09-16T08:21:54.279 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:21:54 smithi028 ceph-mon[64115]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-16T08:21:54.279 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:21:54 smithi028 ceph-mon[64115]: osdmap e105: 8 total, 7 up, 8 in 2024-09-16T08:21:54.279 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:21:54 smithi028 ceph-mon[64115]: from='client.44259 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T08:21:54.279 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:21:54 smithi028 ceph-mon[64115]: from='client.? 172.21.15.28:0/3811324253' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-16T08:21:54.279 INFO:journalctl@ceph.osd.3.smithi028.stdout:Sep 16 08:21:54 smithi028 podman[75716]: 2024-09-16 08:21:54.202443425 +0000 UTC m=+1.889106838 container cleanup dac477d8b04f805426c3b38a4cf7091a8d57debdc1edaa558665de349d16cbe1 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-3, ceph=True, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., maintainer=Guillaume Abrioux , GIT_CLEAN=True, com.redhat.component=centos-stream-container, GIT_BRANCH=HEAD, name=centos-stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, architecture=x86_64, io.openshift.expose-services=, io.k8s.display-name=CentOS Stream 8, version=8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, distribution-scope=public, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_REPO=https://github.com/ceph/ceph-container.git, CEPH_POINT_RELEASE=-17.2.0, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, vcs-type=git, build-date=2022-05-03T08:36:31.336870, io.buildah.version=1.19.8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.tags=base centos centos-stream, RELEASE=HEAD, release=754, vendor=Red Hat, Inc., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754) 2024-09-16T08:21:54.280 INFO:journalctl@ceph.osd.3.smithi028.stdout:Sep 16 08:21:54 smithi028 bash[75716]: ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-3 2024-09-16T08:21:54.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:21:54 smithi144 ceph-mon[49149]: pgmap v158: 97 pgs: 97 active+clean; 351 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 6.5 KiB/s rd, 5.3 MiB/s wr, 527 op/s 2024-09-16T08:21:54.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:21:54 smithi144 ceph-mon[49149]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-16T08:21:54.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:21:54 smithi144 ceph-mon[49149]: osdmap e105: 8 total, 7 up, 8 in 2024-09-16T08:21:54.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:21:54 smithi144 ceph-mon[49149]: from='client.44259 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T08:21:54.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:21:54 smithi144 ceph-mon[49149]: from='client.? 172.21.15.28:0/3811324253' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-16T08:21:54.530 INFO:journalctl@ceph.osd.3.smithi028.stdout:Sep 16 08:21:54 smithi028 podman[75782]: 2024-09-16 08:21:54.463497153 +0000 UTC m=+0.683884337 container remove dac477d8b04f805426c3b38a4cf7091a8d57debdc1edaa558665de349d16cbe1 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-3, io.openshift.tags=base centos centos-stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_REPO=https://github.com/ceph/ceph-container.git, ceph=True, io.k8s.display-name=CentOS Stream 8, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=-17.2.0, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, release=754, GIT_BRANCH=HEAD, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., RELEASE=HEAD, architecture=x86_64, distribution-scope=public, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_CLEAN=True, version=8, com.redhat.component=centos-stream-container, name=centos-stream, vcs-type=git, io.buildah.version=1.19.8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vendor=Red Hat, Inc., build-date=2022-05-03T08:36:31.336870, io.openshift.expose-services=, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac) 2024-09-16T08:21:55.070 INFO:teuthology.orchestra.run.smithi028.stdout: 5 220723 9.90 MB/sec execute 413 sec latency 884.500 ms 2024-09-16T08:21:55.293 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:21:55 smithi028 ceph-mon[64115]: osdmap e106: 8 total, 7 up, 8 in 2024-09-16T08:21:55.293 INFO:journalctl@ceph.osd.3.smithi028.stdout:Sep 16 08:21:55 smithi028 podman[75864]: 2024-09-16 08:21:54.974335731 +0000 UTC m=+0.025057846 image pull 425e316ba8ebf74f04aa3640f7deb4fa3f293a78d0c8d097cb679eaf86ddca6c quay.ceph.io/ceph-ci/ceph@sha256:f7ad55f9ccd17ba66aadb19b4d291e67b60ea24e5d8a0c0e92941fd59c81e4ce 2024-09-16T08:21:55.293 INFO:journalctl@ceph.osd.3.smithi028.stdout:Sep 16 08:21:55 smithi028 podman[75864]: 2024-09-16 08:21:55.139906572 +0000 UTC m=+0.190628673 container create d218ca822edc0fd417ed021bc9a9889fa909cb56b5fa58140ac3ff10d6511ea0 (image=quay.ceph.io/ceph-ci/ceph@sha256:f7ad55f9ccd17ba66aadb19b4d291e67b60ea24e5d8a0c0e92941fd59c81e4ce, name=ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-3-deactivate, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=wip-guits-main-2024-09-13-1248-8293d73, org.label-schema.schema-version=1.0, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_BRANCH=HEAD, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2) 2024-09-16T08:21:55.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:21:55 smithi144 ceph-mon[49149]: osdmap e106: 8 total, 7 up, 8 in 2024-09-16T08:21:55.546 INFO:journalctl@ceph.osd.3.smithi028.stdout:Sep 16 08:21:55 smithi028 podman[75864]: 2024-09-16 08:21:55.51075437 +0000 UTC m=+0.561476484 container init d218ca822edc0fd417ed021bc9a9889fa909cb56b5fa58140ac3ff10d6511ea0 (image=quay.ceph.io/ceph-ci/ceph@sha256:f7ad55f9ccd17ba66aadb19b4d291e67b60ea24e5d8a0c0e92941fd59c81e4ce, name=ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-3-deactivate, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, GIT_CLEAN=True, RELEASE=wip-guits-main-2024-09-13-1248-8293d73, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-16T08:21:55.546 INFO:journalctl@ceph.osd.3.smithi028.stdout:Sep 16 08:21:55 smithi028 podman[75864]: 2024-09-16 08:21:55.523711527 +0000 UTC m=+0.574433634 container start d218ca822edc0fd417ed021bc9a9889fa909cb56b5fa58140ac3ff10d6511ea0 (image=quay.ceph.io/ceph-ci/ceph@sha256:f7ad55f9ccd17ba66aadb19b4d291e67b60ea24e5d8a0c0e92941fd59c81e4ce, name=ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-3-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-guits-main-2024-09-13-1248-8293d73, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, GIT_CLEAN=True, org.label-schema.license=GPLv2) 2024-09-16T08:21:55.798 INFO:journalctl@ceph.osd.3.smithi028.stdout:Sep 16 08:21:55 smithi028 podman[75864]: 2024-09-16 08:21:55.678042482 +0000 UTC m=+0.728764581 container attach d218ca822edc0fd417ed021bc9a9889fa909cb56b5fa58140ac3ff10d6511ea0 (image=quay.ceph.io/ceph-ci/ceph@sha256:f7ad55f9ccd17ba66aadb19b4d291e67b60ea24e5d8a0c0e92941fd59c81e4ce, name=ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-3-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , ceph=True, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_CLEAN=True, RELEASE=wip-guits-main-2024-09-13-1248-8293d73, org.label-schema.vendor=CentOS, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965) 2024-09-16T08:21:56.068 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:21:56 smithi028 ceph-mon[64115]: pgmap v161: 97 pgs: 6 peering, 14 stale+active+clean, 77 active+clean; 350 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 7.2 KiB/s rd, 5.7 MiB/s wr, 584 op/s 2024-09-16T08:21:56.068 INFO:journalctl@ceph.osd.3.smithi028.stdout:Sep 16 08:21:55 smithi028 podman[75864]: 2024-09-16 08:21:55.864330024 +0000 UTC m=+0.915052132 container died d218ca822edc0fd417ed021bc9a9889fa909cb56b5fa58140ac3ff10d6511ea0 (image=quay.ceph.io/ceph-ci/ceph@sha256:f7ad55f9ccd17ba66aadb19b4d291e67b60ea24e5d8a0c0e92941fd59c81e4ce, name=ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-3-deactivate, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , RELEASE=wip-guits-main-2024-09-13-1248-8293d73, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, ceph=True, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS) 2024-09-16T08:21:56.070 INFO:teuthology.orchestra.run.smithi028.stdout: 5 221076 9.89 MB/sec execute 414 sec latency 38.633 ms 2024-09-16T08:21:56.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:21:56 smithi144 ceph-mon[49149]: pgmap v161: 97 pgs: 6 peering, 14 stale+active+clean, 77 active+clean; 350 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 7.2 KiB/s rd, 5.7 MiB/s wr, 584 op/s 2024-09-16T08:21:56.825 INFO:journalctl@ceph.osd.3.smithi028.stdout:Sep 16 08:21:56 smithi028 podman[75887]: 2024-09-16 08:21:56.787185136 +0000 UTC m=+0.914874375 container remove d218ca822edc0fd417ed021bc9a9889fa909cb56b5fa58140ac3ff10d6511ea0 (image=quay.ceph.io/ceph-ci/ceph@sha256:f7ad55f9ccd17ba66aadb19b4d291e67b60ea24e5d8a0c0e92941fd59c81e4ce, name=ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-3-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, RELEASE=wip-guits-main-2024-09-13-1248-8293d73, GIT_CLEAN=True, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=) 2024-09-16T08:21:57.070 INFO:teuthology.orchestra.run.smithi028.stdout: 5 221493 9.90 MB/sec execute 415 sec latency 36.996 ms 2024-09-16T08:21:57.076 INFO:journalctl@ceph.osd.3.smithi028.stdout:Sep 16 08:21:56 smithi028 systemd[1]: ceph-5622b2fe-7402-11ef-bceb-c7b262605968@osd.3.service: Deactivated successfully. 2024-09-16T08:21:57.076 INFO:journalctl@ceph.osd.3.smithi028.stdout:Sep 16 08:21:56 smithi028 systemd[1]: Stopped Ceph osd.3 for 5622b2fe-7402-11ef-bceb-c7b262605968. 2024-09-16T08:21:57.076 INFO:journalctl@ceph.osd.3.smithi028.stdout:Sep 16 08:21:56 smithi028 systemd[1]: ceph-5622b2fe-7402-11ef-bceb-c7b262605968@osd.3.service: Consumed 4min 26.541s CPU time. 2024-09-16T08:21:57.327 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:21:57 smithi028 ceph-mon[64115]: Health check failed: Degraded data redundancy: 59/1662 objects degraded (3.550%), 6 pgs degraded (PG_DEGRADED) 2024-09-16T08:21:57.327 INFO:journalctl@ceph.osd.3.smithi028.stdout:Sep 16 08:21:57 smithi028 systemd[1]: Starting Ceph osd.3 for 5622b2fe-7402-11ef-bceb-c7b262605968... 2024-09-16T08:21:57.578 INFO:journalctl@ceph.osd.3.smithi028.stdout:Sep 16 08:21:57 smithi028 podman[75981]: 2024-09-16 08:21:57.337720927 +0000 UTC m=+0.024049112 image pull 425e316ba8ebf74f04aa3640f7deb4fa3f293a78d0c8d097cb679eaf86ddca6c quay.ceph.io/ceph-ci/ceph@sha256:f7ad55f9ccd17ba66aadb19b4d291e67b60ea24e5d8a0c0e92941fd59c81e4ce 2024-09-16T08:21:57.578 INFO:journalctl@ceph.osd.3.smithi028.stdout:Sep 16 08:21:57 smithi028 podman[75981]: 2024-09-16 08:21:57.531299651 +0000 UTC m=+0.217627831 container create d3974006c74617e712fc24a2502c8a32633703b6b730750df3095980916fbc62 (image=quay.ceph.io/ceph-ci/ceph@sha256:f7ad55f9ccd17ba66aadb19b4d291e67b60ea24e5d8a0c0e92941fd59c81e4ce, name=ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-3-activate, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, ceph=True, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=wip-guits-main-2024-09-13-1248-8293d73) 2024-09-16T08:21:57.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:21:57 smithi144 ceph-mon[49149]: Health check failed: Degraded data redundancy: 59/1662 objects degraded (3.550%), 6 pgs degraded (PG_DEGRADED) 2024-09-16T08:21:57.934 INFO:journalctl@ceph.osd.3.smithi028.stdout:Sep 16 08:21:57 smithi028 podman[75981]: 2024-09-16 08:21:57.909516329 +0000 UTC m=+0.595844519 container init d3974006c74617e712fc24a2502c8a32633703b6b730750df3095980916fbc62 (image=quay.ceph.io/ceph-ci/ceph@sha256:f7ad55f9ccd17ba66aadb19b4d291e67b60ea24e5d8a0c0e92941fd59c81e4ce, name=ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-3-activate, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, RELEASE=wip-guits-main-2024-09-13-1248-8293d73, org.label-schema.license=GPLv2, ceph=True) 2024-09-16T08:21:57.934 INFO:journalctl@ceph.osd.3.smithi028.stdout:Sep 16 08:21:57 smithi028 podman[75981]: 2024-09-16 08:21:57.920397834 +0000 UTC m=+0.606726015 container start d3974006c74617e712fc24a2502c8a32633703b6b730750df3095980916fbc62 (image=quay.ceph.io/ceph-ci/ceph@sha256:f7ad55f9ccd17ba66aadb19b4d291e67b60ea24e5d8a0c0e92941fd59c81e4ce, name=ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-3-activate, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.build-date=20240731, RELEASE=wip-guits-main-2024-09-13-1248-8293d73, ceph=True, GIT_BRANCH=HEAD) 2024-09-16T08:21:58.070 INFO:teuthology.orchestra.run.smithi028.stdout: 5 221919 9.91 MB/sec execute 416 sec latency 33.297 ms 2024-09-16T08:21:58.092 INFO:journalctl@ceph.osd.3.smithi028.stdout:Sep 16 08:21:58 smithi028 podman[75981]: 2024-09-16 08:21:58.068426092 +0000 UTC m=+0.754754279 container attach d3974006c74617e712fc24a2502c8a32633703b6b730750df3095980916fbc62 (image=quay.ceph.io/ceph-ci/ceph@sha256:f7ad55f9ccd17ba66aadb19b4d291e67b60ea24e5d8a0c0e92941fd59c81e4ce, name=ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-3-activate, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, RELEASE=wip-guits-main-2024-09-13-1248-8293d73) 2024-09-16T08:21:58.343 INFO:journalctl@ceph.osd.3.smithi028.stdout:Sep 16 08:21:58 smithi028 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-3-activate[76010]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-16T08:21:58.343 INFO:journalctl@ceph.osd.3.smithi028.stdout:Sep 16 08:21:58 smithi028 bash[75981]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-16T08:21:58.343 INFO:journalctl@ceph.osd.3.smithi028.stdout:Sep 16 08:21:58 smithi028 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-3-activate[76010]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-16T08:21:58.343 INFO:journalctl@ceph.osd.3.smithi028.stdout:Sep 16 08:21:58 smithi028 bash[75981]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-16T08:21:58.343 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:21:58 smithi028 ceph-mon[64115]: pgmap v162: 97 pgs: 3 active+undersized, 6 peering, 11 stale+active+clean, 6 active+undersized+degraded, 71 active+clean; 349 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 7.0 KiB/s rd, 3.2 MiB/s wr, 299 op/s; 59/1662 objects degraded (3.550%) 2024-09-16T08:21:58.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:21:58 smithi144 ceph-mon[49149]: pgmap v162: 97 pgs: 3 active+undersized, 6 peering, 11 stale+active+clean, 6 active+undersized+degraded, 71 active+clean; 349 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 7.0 KiB/s rd, 3.2 MiB/s wr, 299 op/s; 59/1662 objects degraded (3.550%) 2024-09-16T08:21:59.070 INFO:teuthology.orchestra.run.smithi028.stdout: 5 222296 9.92 MB/sec execute 417 sec latency 48.968 ms 2024-09-16T08:22:00.070 INFO:teuthology.orchestra.run.smithi028.stdout: 5 222650 9.93 MB/sec execute 418 sec latency 69.132 ms 2024-09-16T08:22:00.128 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:22:00 smithi028 ceph-mon[64115]: pgmap v163: 97 pgs: 13 active+undersized, 6 peering, 22 active+undersized+degraded, 56 active+clean; 354 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 5.4 MiB/s wr, 568 op/s; 193/1659 objects degraded (11.634%) 2024-09-16T08:22:00.383 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:22:00 smithi144 ceph-mon[49149]: pgmap v163: 97 pgs: 13 active+undersized, 6 peering, 22 active+undersized+degraded, 56 active+clean; 354 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 5.4 MiB/s wr, 568 op/s; 193/1659 objects degraded (11.634%) 2024-09-16T08:22:01.070 INFO:teuthology.orchestra.run.smithi028.stdout: 5 222905 9.91 MB/sec execute 419 sec latency 75.375 ms 2024-09-16T08:22:01.146 INFO:journalctl@ceph.osd.3.smithi028.stdout:Sep 16 08:22:01 smithi028 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-3-activate[76010]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-09-16T08:22:01.146 INFO:journalctl@ceph.osd.3.smithi028.stdout:Sep 16 08:22:01 smithi028 bash[75981]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-09-16T08:22:01.146 INFO:journalctl@ceph.osd.3.smithi028.stdout:Sep 16 08:22:01 smithi028 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-3-activate[76010]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-3 --no-mon-config --dev /dev/mapper/vg_nvme-lv_1 2024-09-16T08:22:01.146 INFO:journalctl@ceph.osd.3.smithi028.stdout:Sep 16 08:22:01 smithi028 bash[75981]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-3 --no-mon-config --dev /dev/mapper/vg_nvme-lv_1 2024-09-16T08:22:01.665 INFO:journalctl@ceph.osd.3.smithi028.stdout:Sep 16 08:22:01 smithi028 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-3-activate[76010]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_1 2024-09-16T08:22:01.665 INFO:journalctl@ceph.osd.3.smithi028.stdout:Sep 16 08:22:01 smithi028 bash[75981]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_1 2024-09-16T08:22:01.665 INFO:journalctl@ceph.osd.3.smithi028.stdout:Sep 16 08:22:01 smithi028 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-3-activate[76010]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-09-16T08:22:01.665 INFO:journalctl@ceph.osd.3.smithi028.stdout:Sep 16 08:22:01 smithi028 bash[75981]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-09-16T08:22:01.665 INFO:journalctl@ceph.osd.3.smithi028.stdout:Sep 16 08:22:01 smithi028 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-3-activate[76010]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-3/block 2024-09-16T08:22:01.665 INFO:journalctl@ceph.osd.3.smithi028.stdout:Sep 16 08:22:01 smithi028 bash[75981]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-3/block 2024-09-16T08:22:01.666 INFO:journalctl@ceph.osd.3.smithi028.stdout:Sep 16 08:22:01 smithi028 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-3-activate[76010]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-09-16T08:22:01.666 INFO:journalctl@ceph.osd.3.smithi028.stdout:Sep 16 08:22:01 smithi028 bash[75981]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-09-16T08:22:01.666 INFO:journalctl@ceph.osd.3.smithi028.stdout:Sep 16 08:22:01 smithi028 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-3-activate[76010]: --> ceph-volume raw activate successful for osd ID: 3 2024-09-16T08:22:01.666 INFO:journalctl@ceph.osd.3.smithi028.stdout:Sep 16 08:22:01 smithi028 bash[75981]: --> ceph-volume raw activate successful for osd ID: 3 2024-09-16T08:22:01.666 INFO:journalctl@ceph.osd.3.smithi028.stdout:Sep 16 08:22:01 smithi028 podman[75981]: 2024-09-16 08:22:01.531433359 +0000 UTC m=+4.217761540 container died d3974006c74617e712fc24a2502c8a32633703b6b730750df3095980916fbc62 (image=quay.ceph.io/ceph-ci/ceph@sha256:f7ad55f9ccd17ba66aadb19b4d291e67b60ea24e5d8a0c0e92941fd59c81e4ce, name=ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-3-activate, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=wip-guits-main-2024-09-13-1248-8293d73, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=) 2024-09-16T08:22:02.070 INFO:teuthology.orchestra.run.smithi028.stdout: 5 223064 9.89 MB/sec execute 420 sec latency 92.690 ms 2024-09-16T08:22:02.203 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:22:02 smithi028 ceph-mon[64115]: pgmap v164: 97 pgs: 15 active+undersized, 26 active+undersized+degraded, 56 active+clean; 353 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 5.6 KiB/s rd, 4.9 MiB/s wr, 492 op/s; 224/1698 objects degraded (13.192%) 2024-09-16T08:22:02.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:22:02 smithi144 ceph-mon[49149]: pgmap v164: 97 pgs: 15 active+undersized, 26 active+undersized+degraded, 56 active+clean; 353 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 5.6 KiB/s rd, 4.9 MiB/s wr, 492 op/s; 224/1698 objects degraded (13.192%) 2024-09-16T08:22:02.727 INFO:journalctl@ceph.osd.3.smithi028.stdout:Sep 16 08:22:02 smithi028 podman[76168]: 2024-09-16 08:22:02.504530574 +0000 UTC m=+0.957875050 container remove d3974006c74617e712fc24a2502c8a32633703b6b730750df3095980916fbc62 (image=quay.ceph.io/ceph-ci/ceph@sha256:f7ad55f9ccd17ba66aadb19b4d291e67b60ea24e5d8a0c0e92941fd59c81e4ce, name=ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-3-activate, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, ceph=True, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, RELEASE=wip-guits-main-2024-09-13-1248-8293d73, GIT_BRANCH=HEAD) 2024-09-16T08:22:02.982 INFO:journalctl@ceph.osd.3.smithi028.stdout:Sep 16 08:22:02 smithi028 podman[76227]: 2024-09-16 08:22:02.637330596 +0000 UTC m=+0.019070227 image pull 425e316ba8ebf74f04aa3640f7deb4fa3f293a78d0c8d097cb679eaf86ddca6c quay.ceph.io/ceph-ci/ceph@sha256:f7ad55f9ccd17ba66aadb19b4d291e67b60ea24e5d8a0c0e92941fd59c81e4ce 2024-09-16T08:22:02.982 INFO:journalctl@ceph.osd.3.smithi028.stdout:Sep 16 08:22:02 smithi028 podman[76227]: 2024-09-16 08:22:02.736210359 +0000 UTC m=+0.117949980 container create bd84ba1224f2d5e5f2a6a62fb7ed5cb90fbe74581f2807beaa2f9af07a91afb2 (image=quay.ceph.io/ceph-ci/ceph@sha256:f7ad55f9ccd17ba66aadb19b4d291e67b60ea24e5d8a0c0e92941fd59c81e4ce, name=ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-3, RELEASE=wip-guits-main-2024-09-13-1248-8293d73, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, ceph=True, org.label-schema.build-date=20240731) 2024-09-16T08:22:03.070 INFO:teuthology.orchestra.run.smithi028.stdout: 5 223248 9.87 MB/sec execute 421 sec latency 66.187 ms 2024-09-16T08:22:03.256 INFO:journalctl@ceph.osd.3.smithi028.stdout:Sep 16 08:22:03 smithi028 podman[76227]: 2024-09-16 08:22:03.123730506 +0000 UTC m=+0.505470148 container init bd84ba1224f2d5e5f2a6a62fb7ed5cb90fbe74581f2807beaa2f9af07a91afb2 (image=quay.ceph.io/ceph-ci/ceph@sha256:f7ad55f9ccd17ba66aadb19b4d291e67b60ea24e5d8a0c0e92941fd59c81e4ce, name=ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-3, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, RELEASE=wip-guits-main-2024-09-13-1248-8293d73, GIT_CLEAN=True, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_BRANCH=HEAD) 2024-09-16T08:22:03.256 INFO:journalctl@ceph.osd.3.smithi028.stdout:Sep 16 08:22:03 smithi028 podman[76227]: 2024-09-16 08:22:03.137383476 +0000 UTC m=+0.519123097 container start bd84ba1224f2d5e5f2a6a62fb7ed5cb90fbe74581f2807beaa2f9af07a91afb2 (image=quay.ceph.io/ceph-ci/ceph@sha256:f7ad55f9ccd17ba66aadb19b4d291e67b60ea24e5d8a0c0e92941fd59c81e4ce, name=ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-3, RELEASE=wip-guits-main-2024-09-13-1248-8293d73, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.build-date=20240731, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-16T08:22:03.256 INFO:journalctl@ceph.osd.3.smithi028.stdout:Sep 16 08:22:03 smithi028 ceph-osd[76245]: -- 172.21.15.28:0/3798778371 <== mon.0 v2:172.21.15.28:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x56396853c1e0 con 0x5639676d7c00 2024-09-16T08:22:03.256 INFO:journalctl@ceph.osd.3.smithi028.stdout:Sep 16 08:22:03 smithi028 bash[76227]: bd84ba1224f2d5e5f2a6a62fb7ed5cb90fbe74581f2807beaa2f9af07a91afb2 2024-09-16T08:22:03.257 INFO:journalctl@ceph.osd.3.smithi028.stdout:Sep 16 08:22:03 smithi028 systemd[1]: Started Ceph osd.3 for 5622b2fe-7402-11ef-bceb-c7b262605968. 2024-09-16T08:22:03.524 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:22:03 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:22:03.524 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:22:03 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-16T08:22:03.524 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:22:03 smithi028 ceph-mon[64115]: pgmap v165: 97 pgs: 15 active+undersized, 26 active+undersized+degraded, 56 active+clean; 352 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 4.6 KiB/s rd, 4.3 MiB/s wr, 385 op/s; 222/1689 objects degraded (13.144%) 2024-09-16T08:22:03.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:22:03 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:22:03.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:22:03 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-16T08:22:03.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:22:03 smithi144 ceph-mon[49149]: pgmap v165: 97 pgs: 15 active+undersized, 26 active+undersized+degraded, 56 active+clean; 352 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 4.6 KiB/s rd, 4.3 MiB/s wr, 385 op/s; 222/1689 objects degraded (13.144%) 2024-09-16T08:22:04.072 INFO:teuthology.orchestra.run.smithi028.stdout: 5 223519 9.87 MB/sec execute 422 sec latency 66.325 ms 2024-09-16T08:22:04.296 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:22:04 smithi028 ceph-mon[64115]: Health check update: Degraded data redundancy: 222/1689 objects degraded (13.144%), 26 pgs degraded (PG_DEGRADED) 2024-09-16T08:22:04.296 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:22:04 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:22:04.296 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:22:04 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:22:04.297 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:22:04 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-16T08:22:04.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:22:04 smithi144 ceph-mon[49149]: Health check update: Degraded data redundancy: 222/1689 objects degraded (13.144%), 26 pgs degraded (PG_DEGRADED) 2024-09-16T08:22:04.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:22:04 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:22:04.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:22:04 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:22:04.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:22:04 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-16T08:22:05.049 INFO:journalctl@ceph.osd.3.smithi028.stdout:Sep 16 08:22:05 smithi028 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-3[76241]: 2024-09-16T08:22:05.016+0000 7fd990b5e640 -1 Falling back to public interface 2024-09-16T08:22:05.071 INFO:teuthology.orchestra.run.smithi028.stdout: 5 224066 9.88 MB/sec execute 423 sec latency 32.250 ms 2024-09-16T08:22:05.298 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:22:05 smithi028 ceph-mon[64115]: pgmap v166: 97 pgs: 15 active+undersized, 26 active+undersized+degraded, 56 active+clean; 351 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 5.8 KiB/s rd, 4.4 MiB/s wr, 432 op/s; 224/1677 objects degraded (13.357%) 2024-09-16T08:22:05.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:22:05 smithi144 ceph-mon[49149]: pgmap v166: 97 pgs: 15 active+undersized, 26 active+undersized+degraded, 56 active+clean; 351 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 5.8 KiB/s rd, 4.4 MiB/s wr, 432 op/s; 224/1677 objects degraded (13.357%) 2024-09-16T08:22:06.070 INFO:teuthology.orchestra.run.smithi028.stdout: 5 224613 9.88 MB/sec execute 424 sec latency 31.507 ms 2024-09-16T08:22:07.071 INFO:teuthology.orchestra.run.smithi028.stdout: 5 225104 9.89 MB/sec execute 425 sec latency 28.290 ms 2024-09-16T08:22:08.040 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:22:07 smithi028 ceph-mon[64115]: pgmap v167: 97 pgs: 15 active+undersized, 26 active+undersized+degraded, 56 active+clean; 350 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 5.2 KiB/s rd, 4.2 MiB/s wr, 408 op/s; 224/1671 objects degraded (13.405%) 2024-09-16T08:22:08.071 INFO:teuthology.orchestra.run.smithi028.stdout: 5 225451 9.90 MB/sec execute 426 sec latency 52.332 ms 2024-09-16T08:22:08.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:22:07 smithi144 ceph-mon[49149]: pgmap v167: 97 pgs: 15 active+undersized, 26 active+undersized+degraded, 56 active+clean; 350 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 5.2 KiB/s rd, 4.2 MiB/s wr, 408 op/s; 224/1671 objects degraded (13.405%) 2024-09-16T08:22:09.071 INFO:teuthology.orchestra.run.smithi028.stdout: 5 225714 9.90 MB/sec execute 427 sec latency 70.510 ms 2024-09-16T08:22:09.077 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:22:08 smithi028 ceph-mon[64115]: Health check update: Degraded data redundancy: 227/1707 objects degraded (13.298%), 26 pgs degraded (PG_DEGRADED) 2024-09-16T08:22:09.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:22:08 smithi144 ceph-mon[49149]: Health check update: Degraded data redundancy: 227/1707 objects degraded (13.298%), 26 pgs degraded (PG_DEGRADED) 2024-09-16T08:22:10.071 INFO:teuthology.orchestra.run.smithi028.stdout: 5 225991 9.91 MB/sec execute 428 sec latency 73.232 ms 2024-09-16T08:22:10.102 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:22:09 smithi028 ceph-mon[64115]: pgmap v168: 97 pgs: 15 active+undersized, 26 active+undersized+degraded, 56 active+clean; 356 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 5.4 KiB/s rd, 6.1 MiB/s wr, 626 op/s; 227/1707 objects degraded (13.298%) 2024-09-16T08:22:10.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:22:09 smithi144 ceph-mon[49149]: pgmap v168: 97 pgs: 15 active+undersized, 26 active+undersized+degraded, 56 active+clean; 356 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 5.4 KiB/s rd, 6.1 MiB/s wr, 626 op/s; 227/1707 objects degraded (13.298%) 2024-09-16T08:22:11.071 INFO:teuthology.orchestra.run.smithi028.stdout: 5 226315 9.90 MB/sec execute 429 sec latency 42.705 ms 2024-09-16T08:22:12.071 INFO:teuthology.orchestra.run.smithi028.stdout: 5 226580 9.88 MB/sec execute 430 sec latency 59.954 ms 2024-09-16T08:22:12.148 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:22:11 smithi028 ceph-mon[64115]: pgmap v169: 97 pgs: 15 active+undersized, 26 active+undersized+degraded, 56 active+clean; 348 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 9.7 KiB/s rd, 4.8 MiB/s wr, 460 op/s; 222/1683 objects degraded (13.191%) 2024-09-16T08:22:12.148 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:22:11 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:22:12.148 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:22:11 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:22:12.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:22:11 smithi144 ceph-mon[49149]: pgmap v169: 97 pgs: 15 active+undersized, 26 active+undersized+degraded, 56 active+clean; 348 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 9.7 KiB/s rd, 4.8 MiB/s wr, 460 op/s; 222/1683 objects degraded (13.191%) 2024-09-16T08:22:12.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:22:11 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:22:12.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:22:11 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:22:12.655 INFO:journalctl@ceph.osd.3.smithi028.stdout:Sep 16 08:22:12 smithi028 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-3[76241]: 2024-09-16T08:22:12.397+0000 7fd990b5e640 -1 osd.3 0 read_superblock omap replica is missing. 2024-09-16T08:22:13.071 INFO:teuthology.orchestra.run.smithi028.stdout: 5 226788 9.86 MB/sec execute 431 sec latency 56.519 ms 2024-09-16T08:22:13.962 INFO:journalctl@ceph.osd.3.smithi028.stdout:Sep 16 08:22:13 smithi028 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-3[76241]: 2024-09-16T08:22:13.818+0000 7fd990b5e640 -1 osd.3 104 log_to_monitors true 2024-09-16T08:22:13.962 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:22:13 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:22:13.963 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:22:13 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:22:13.963 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:22:13 smithi028 ceph-mon[64115]: pgmap v170: 97 pgs: 15 active+undersized, 26 active+undersized+degraded, 56 active+clean; 356 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 6.6 KiB/s rd, 5.6 MiB/s wr, 579 op/s; 228/1719 objects degraded (13.264%) 2024-09-16T08:22:14.071 INFO:teuthology.orchestra.run.smithi028.stdout: 5 227030 9.86 MB/sec execute 432 sec latency 63.203 ms 2024-09-16T08:22:14.081 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:22:13 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:22:14.081 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:22:13 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:22:14.082 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:22:13 smithi144 ceph-mon[49149]: pgmap v170: 97 pgs: 15 active+undersized, 26 active+undersized+degraded, 56 active+clean; 356 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 6.6 KiB/s rd, 5.6 MiB/s wr, 579 op/s; 228/1719 objects degraded (13.264%) 2024-09-16T08:22:15.000 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:22:14 smithi028 ceph-mon[64115]: from='osd.3 [v2:172.21.15.28:6826/573940536,v1:172.21.15.28:6827/573940536]' entity='osd.3' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]} : dispatch 2024-09-16T08:22:15.071 INFO:teuthology.orchestra.run.smithi028.stdout: 5 227277 9.86 MB/sec execute 433 sec latency 59.521 ms 2024-09-16T08:22:15.081 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:22:14 smithi144 ceph-mon[49149]: from='osd.3 [v2:172.21.15.28:6826/573940536,v1:172.21.15.28:6827/573940536]' entity='osd.3' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]} : dispatch 2024-09-16T08:22:15.256 INFO:journalctl@ceph.osd.3.smithi028.stdout:Sep 16 08:22:15 smithi028 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-3[76241]: 2024-09-16T08:22:15.087+0000 7fd984900640 -1 osd.3 104 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-16T08:22:15.784 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:22:15 smithi028 ceph-mon[64115]: from='osd.3 [v2:172.21.15.28:6826/573940536,v1:172.21.15.28:6827/573940536]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-09-16T08:22:15.784 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:22:15 smithi028 ceph-mon[64115]: osdmap e107: 8 total, 7 up, 8 in 2024-09-16T08:22:15.784 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:22:15 smithi028 ceph-mon[64115]: from='osd.3 [v2:172.21.15.28:6826/573940536,v1:172.21.15.28:6827/573940536]' entity='osd.3' cmd={"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi028", "root=default"]} : dispatch 2024-09-16T08:22:15.784 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:22:15 smithi028 ceph-mon[64115]: pgmap v172: 97 pgs: 15 active+undersized, 26 active+undersized+degraded, 56 active+clean; 352 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.6 KiB/s rd, 5.2 MiB/s wr, 516 op/s; 223/1695 objects degraded (13.156%) 2024-09-16T08:22:15.784 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:22:15 smithi028 ceph-mon[64115]: Health check update: Degraded data redundancy: 228/1719 objects degraded (13.264%), 26 pgs degraded (PG_DEGRADED) 2024-09-16T08:22:15.784 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:22:15 smithi028 ceph-mon[64115]: from='osd.3 [v2:172.21.15.28:6826/573940536,v1:172.21.15.28:6827/573940536]' entity='osd.3' 2024-09-16T08:22:16.071 INFO:teuthology.orchestra.run.smithi028.stdout: 5 227520 9.84 MB/sec execute 434 sec latency 81.430 ms 2024-09-16T08:22:16.082 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:22:15 smithi144 ceph-mon[49149]: from='osd.3 [v2:172.21.15.28:6826/573940536,v1:172.21.15.28:6827/573940536]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-09-16T08:22:16.082 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:22:15 smithi144 ceph-mon[49149]: osdmap e107: 8 total, 7 up, 8 in 2024-09-16T08:22:16.082 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:22:15 smithi144 ceph-mon[49149]: from='osd.3 [v2:172.21.15.28:6826/573940536,v1:172.21.15.28:6827/573940536]' entity='osd.3' cmd={"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi028", "root=default"]} : dispatch 2024-09-16T08:22:16.082 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:22:15 smithi144 ceph-mon[49149]: pgmap v172: 97 pgs: 15 active+undersized, 26 active+undersized+degraded, 56 active+clean; 352 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.6 KiB/s rd, 5.2 MiB/s wr, 516 op/s; 223/1695 objects degraded (13.156%) 2024-09-16T08:22:16.082 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:22:15 smithi144 ceph-mon[49149]: Health check update: Degraded data redundancy: 228/1719 objects degraded (13.264%), 26 pgs degraded (PG_DEGRADED) 2024-09-16T08:22:16.082 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:22:15 smithi144 ceph-mon[49149]: from='osd.3 [v2:172.21.15.28:6826/573940536,v1:172.21.15.28:6827/573940536]' entity='osd.3' 2024-09-16T08:22:17.071 INFO:teuthology.orchestra.run.smithi028.stdout: 5 227544 9.82 MB/sec execute 435 sec latency 1015.444 ms 2024-09-16T08:22:17.082 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:22:16 smithi144 ceph-mon[49149]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-16T08:22:17.082 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:22:16 smithi144 ceph-mon[49149]: osd.3 [v2:172.21.15.28:6826/573940536,v1:172.21.15.28:6827/573940536] boot 2024-09-16T08:22:17.082 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:22:16 smithi144 ceph-mon[49149]: osdmap e108: 8 total, 8 up, 8 in 2024-09-16T08:22:17.082 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:22:16 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "osd metadata", "id": 3} : dispatch 2024-09-16T08:22:17.197 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:22:16 smithi028 ceph-mon[64115]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-16T08:22:17.197 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:22:16 smithi028 ceph-mon[64115]: osd.3 [v2:172.21.15.28:6826/573940536,v1:172.21.15.28:6827/573940536] boot 2024-09-16T08:22:17.197 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:22:16 smithi028 ceph-mon[64115]: osdmap e108: 8 total, 8 up, 8 in 2024-09-16T08:22:17.197 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:22:16 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "osd metadata", "id": 3} : dispatch 2024-09-16T08:22:18.071 INFO:teuthology.orchestra.run.smithi028.stdout: 5 227544 9.80 MB/sec execute 436 sec latency 2015.483 ms 2024-09-16T08:22:18.324 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:22:18 smithi028 ceph-mon[64115]: pgmap v174: 97 pgs: 15 active+undersized, 26 active+undersized+degraded, 56 active+clean; 351 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 9.2 KiB/s rd, 3.4 MiB/s wr, 295 op/s; 218/1689 objects degraded (12.907%) 2024-09-16T08:22:18.324 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:22:18 smithi028 ceph-mon[64115]: osdmap e109: 8 total, 8 up, 8 in 2024-09-16T08:22:18.324 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:22:18 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:22:18.324 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:22:18 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-16T08:22:18.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:22:18 smithi144 ceph-mon[49149]: pgmap v174: 97 pgs: 15 active+undersized, 26 active+undersized+degraded, 56 active+clean; 351 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 9.2 KiB/s rd, 3.4 MiB/s wr, 295 op/s; 218/1689 objects degraded (12.907%) 2024-09-16T08:22:18.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:22:18 smithi144 ceph-mon[49149]: osdmap e109: 8 total, 8 up, 8 in 2024-09-16T08:22:18.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:22:18 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:22:18.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:22:18 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-16T08:22:19.071 INFO:teuthology.orchestra.run.smithi028.stdout: 5 227824 9.79 MB/sec execute 437 sec latency 2129.731 ms 2024-09-16T08:22:19.097 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:22:19 smithi028 ceph-mon[64115]: osdmap e110: 8 total, 8 up, 8 in 2024-09-16T08:22:19.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:22:19 smithi144 ceph-mon[49149]: osdmap e110: 8 total, 8 up, 8 in 2024-09-16T08:22:20.071 INFO:teuthology.orchestra.run.smithi028.stdout: 5 228069 9.78 MB/sec execute 438 sec latency 71.523 ms 2024-09-16T08:22:20.123 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:22:20 smithi028 ceph-mon[64115]: pgmap v177: 97 pgs: 6 remapped+peering, 9 active+undersized, 15 active+undersized+degraded, 67 active+clean; 348 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 3.1 MiB/s wr, 334 op/s; 112/1551 objects degraded (7.221%); 21 B/s, 0 objects/s recovering 2024-09-16T08:22:20.123 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:22:20 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:22:20.123 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:22:20 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:22:20.124 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:22:20 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-16T08:22:20.124 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:22:20 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-16T08:22:20.124 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:22:20 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:22:20.124 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:22:20 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:22:20.124 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:22:20 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-16T08:22:20.124 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:22:20 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "versions"} : dispatch 2024-09-16T08:22:20.124 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:22:20 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "versions"} : dispatch 2024-09-16T08:22:20.124 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:22:20 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "versions"} : dispatch 2024-09-16T08:22:20.124 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:22:20 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16} : dispatch 2024-09-16T08:22:20.124 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:22:20 smithi028 ceph-mon[64115]: Health check update: Degraded data redundancy: 112/1551 objects degraded (7.221%), 15 pgs degraded (PG_DEGRADED) 2024-09-16T08:22:20.124 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:22:20 smithi028 ceph-mon[64115]: osdmap e111: 8 total, 8 up, 8 in 2024-09-16T08:22:20.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:22:20 smithi144 ceph-mon[49149]: pgmap v177: 97 pgs: 6 remapped+peering, 9 active+undersized, 15 active+undersized+degraded, 67 active+clean; 348 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 3.1 MiB/s wr, 334 op/s; 112/1551 objects degraded (7.221%); 21 B/s, 0 objects/s recovering 2024-09-16T08:22:20.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:22:20 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:22:20.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:22:20 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:22:20.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:22:20 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-16T08:22:20.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:22:20 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-16T08:22:20.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:22:20 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:22:20.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:22:20 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:22:20.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:22:20 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-16T08:22:20.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:22:20 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "versions"} : dispatch 2024-09-16T08:22:20.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:22:20 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "versions"} : dispatch 2024-09-16T08:22:20.583 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:22:20 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "versions"} : dispatch 2024-09-16T08:22:20.583 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:22:20 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16} : dispatch 2024-09-16T08:22:20.583 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:22:20 smithi144 ceph-mon[49149]: Health check update: Degraded data redundancy: 112/1551 objects degraded (7.221%), 15 pgs degraded (PG_DEGRADED) 2024-09-16T08:22:20.583 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:22:20 smithi144 ceph-mon[49149]: osdmap e111: 8 total, 8 up, 8 in 2024-09-16T08:22:21.071 INFO:teuthology.orchestra.run.smithi028.stdout: 5 228299 9.77 MB/sec execute 439 sec latency 813.918 ms 2024-09-16T08:22:21.144 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:22:21 smithi028 ceph-mon[64115]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-09-16T08:22:21.144 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:22:21 smithi028 ceph-mon[64115]: Upgrade: unsafe to stop osd(s) at this time (7 PGs are or would become offline) 2024-09-16T08:22:21.144 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:22:21 smithi028 ceph-mon[64115]: osdmap e112: 8 total, 8 up, 8 in 2024-09-16T08:22:21.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:22:21 smithi144 ceph-mon[49149]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-09-16T08:22:21.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:22:21 smithi144 ceph-mon[49149]: Upgrade: unsafe to stop osd(s) at this time (7 PGs are or would become offline) 2024-09-16T08:22:21.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:22:21 smithi144 ceph-mon[49149]: osdmap e112: 8 total, 8 up, 8 in 2024-09-16T08:22:22.071 INFO:teuthology.orchestra.run.smithi028.stdout: 5 228675 9.77 MB/sec execute 440 sec latency 151.522 ms 2024-09-16T08:22:22.176 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:22:22 smithi028 ceph-mon[64115]: pgmap v179: 97 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 6 remapped+peering, 85 active+clean; 343 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 4.0 KiB/s rd, 2.3 MiB/s wr, 280 op/s; 151/1518 objects degraded (9.947%); 29 KiB/s, 4 objects/s recovering 2024-09-16T08:22:22.176 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:22:22 smithi028 ceph-mon[64115]: osdmap e113: 8 total, 8 up, 8 in 2024-09-16T08:22:22.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:22:22 smithi144 ceph-mon[49149]: pgmap v179: 97 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 6 remapped+peering, 85 active+clean; 343 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 4.0 KiB/s rd, 2.3 MiB/s wr, 280 op/s; 151/1518 objects degraded (9.947%); 29 KiB/s, 4 objects/s recovering 2024-09-16T08:22:22.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:22:22 smithi144 ceph-mon[49149]: osdmap e113: 8 total, 8 up, 8 in 2024-09-16T08:22:23.071 INFO:teuthology.orchestra.run.smithi028.stdout: 5 228817 9.76 MB/sec execute 441 sec latency 905.753 ms 2024-09-16T08:22:23.963 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:22:23 smithi028 ceph-mon[64115]: pgmap v182: 97 pgs: 1 peering, 6 active+recovery_wait+undersized+degraded+remapped, 3 remapped+peering, 87 active+clean; 343 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 4.0 KiB/s rd, 1.9 MiB/s wr, 180 op/s; 151/1584 objects degraded (9.533%); 44 KiB/s, 20 objects/s recovering 2024-09-16T08:22:23.963 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:22:23 smithi028 ceph-mon[64115]: osdmap e114: 8 total, 8 up, 8 in 2024-09-16T08:22:24.071 INFO:teuthology.orchestra.run.smithi028.stdout: 5 229141 9.76 MB/sec execute 442 sec latency 864.611 ms 2024-09-16T08:22:24.304 INFO:teuthology.orchestra.run.smithi028.stdout:true 2024-09-16T08:22:24.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:22:23 smithi144 ceph-mon[49149]: pgmap v182: 97 pgs: 1 peering, 6 active+recovery_wait+undersized+degraded+remapped, 3 remapped+peering, 87 active+clean; 343 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 4.0 KiB/s rd, 1.9 MiB/s wr, 180 op/s; 151/1584 objects degraded (9.533%); 44 KiB/s, 20 objects/s recovering 2024-09-16T08:22:24.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:22:23 smithi144 ceph-mon[49149]: osdmap e114: 8 total, 8 up, 8 in 2024-09-16T08:22:24.971 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:22:24 smithi028 ceph-mon[64115]: osdmap e115: 8 total, 8 up, 8 in 2024-09-16T08:22:24.971 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:22:24 smithi028 ceph-mon[64115]: Health check update: Degraded data redundancy: 151/1584 objects degraded (9.533%), 6 pgs degraded (PG_DEGRADED) 2024-09-16T08:22:25.071 INFO:teuthology.orchestra.run.smithi028.stdout: 5 229509 9.77 MB/sec execute 443 sec latency 49.900 ms 2024-09-16T08:22:25.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:22:24 smithi144 ceph-mon[49149]: osdmap e115: 8 total, 8 up, 8 in 2024-09-16T08:22:25.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:22:24 smithi144 ceph-mon[49149]: Health check update: Degraded data redundancy: 151/1584 objects degraded (9.533%), 6 pgs degraded (PG_DEGRADED) 2024-09-16T08:22:25.458 INFO:teuthology.orchestra.run.smithi028.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-16T08:22:25.458 INFO:teuthology.orchestra.run.smithi028.stdout:alertmanager.smithi028 smithi028 *:9093,9094 running (6m) 13s ago 15m 18.6M - 0.27.0 11f11916f8cd a14fcb4cb2b6 2024-09-16T08:22:25.458 INFO:teuthology.orchestra.run.smithi028.stdout:crash.smithi028 smithi028 running (3m) 13s ago 15m 6899k - 19.3.0-4917-g8293d73f 425e316ba8eb 4bb89df2a867 2024-09-16T08:22:25.458 INFO:teuthology.orchestra.run.smithi028.stdout:crash.smithi144 smithi144 running (3m) 3m ago 15m 6895k - 19.3.0-4917-g8293d73f 425e316ba8eb 24a96875d7af 2024-09-16T08:22:25.458 INFO:teuthology.orchestra.run.smithi028.stdout:grafana.smithi028 smithi028 *:3000 running (6m) 13s ago 15m 45.9M - dad864ee21e9 b55ba8b944a1 2024-09-16T08:22:25.458 INFO:teuthology.orchestra.run.smithi028.stdout:mds.foofs.smithi028.srtgev smithi028 running (9m) 13s ago 9m 38.5M - 17.2.0 e1d6a67b021e 250e7a7f754c 2024-09-16T08:22:25.459 INFO:teuthology.orchestra.run.smithi028.stdout:mds.foofs.smithi144.oozgsi smithi144 running (9m) 3m ago 9m 16.6M - 17.2.0 e1d6a67b021e ecf4c2bb0fdc 2024-09-16T08:22:25.459 INFO:teuthology.orchestra.run.smithi028.stdout:mgr.smithi028.azvyzy smithi028 *:8443,9283,8765 running (5m) 13s ago 16m 519M - 19.3.0-4917-g8293d73f 425e316ba8eb e01d4382d03f 2024-09-16T08:22:25.459 INFO:teuthology.orchestra.run.smithi028.stdout:mgr.smithi144.utzszj smithi144 *:8443,9283,8765 running (5m) 3m ago 15m 453M - 19.3.0-4917-g8293d73f 425e316ba8eb 71f7f802bffb 2024-09-16T08:22:25.459 INFO:teuthology.orchestra.run.smithi028.stdout:mon.smithi028 smithi028 running (4m) 13s ago 17m 49.8M 2048M 19.3.0-4917-g8293d73f 425e316ba8eb 14efcedc0088 2024-09-16T08:22:25.459 INFO:teuthology.orchestra.run.smithi028.stdout:mon.smithi144 smithi144 running (4m) 3m ago 15m 38.1M 2048M 19.3.0-4917-g8293d73f 425e316ba8eb c081c5ed44b5 2024-09-16T08:22:25.459 INFO:teuthology.orchestra.run.smithi028.stdout:nfs.foo.0.0.smithi028.dkttgj smithi028 *:2049 running (9m) 13s ago 9m 285M - 4.0 e1d6a67b021e b5dccc45a6ed 2024-09-16T08:22:25.459 INFO:teuthology.orchestra.run.smithi028.stdout:nfs.foo.1.0.smithi144.epipkg smithi144 *:2049 running (9m) 3m ago 9m 87.0M - 4.0 e1d6a67b021e faa89e61f50e 2024-09-16T08:22:25.459 INFO:teuthology.orchestra.run.smithi028.stdout:node-exporter.smithi028 smithi028 *:9100 running (6m) 13s ago 15m 9.92M - 1.7.0 72c9c2088986 9e2b8e8a63a9 2024-09-16T08:22:25.459 INFO:teuthology.orchestra.run.smithi028.stdout:node-exporter.smithi144 smithi144 *:9100 running (6m) 3m ago 14m 10.1M - 1.7.0 72c9c2088986 78834c5e992f 2024-09-16T08:22:25.459 INFO:teuthology.orchestra.run.smithi028.stdout:osd.0 smithi028 running (2m) 13s ago 14m 354M 1937M 19.3.0-4917-g8293d73f 425e316ba8eb 2fc02ba4f14f 2024-09-16T08:22:25.459 INFO:teuthology.orchestra.run.smithi028.stdout:osd.1 smithi028 running (2m) 13s ago 13m 266M 1937M 19.3.0-4917-g8293d73f 425e316ba8eb df34bd603d8f 2024-09-16T08:22:25.459 INFO:teuthology.orchestra.run.smithi028.stdout:osd.2 smithi028 running (72s) 13s ago 13m 220M 1937M 19.3.0-4917-g8293d73f 425e316ba8eb c92be6327b12 2024-09-16T08:22:25.460 INFO:teuthology.orchestra.run.smithi028.stdout:osd.3 smithi028 running (22s) 13s ago 13m 11.2M 1937M 19.3.0-4917-g8293d73f 425e316ba8eb bd84ba1224f2 2024-09-16T08:22:25.460 INFO:teuthology.orchestra.run.smithi028.stdout:osd.4 smithi144 running (12m) 3m ago 12m 527M 2704M 17.2.0 e1d6a67b021e 86719c60975c 2024-09-16T08:22:25.460 INFO:teuthology.orchestra.run.smithi028.stdout:osd.5 smithi144 running (12m) 3m ago 12m 334M 2704M 17.2.0 e1d6a67b021e 70452c145c60 2024-09-16T08:22:25.460 INFO:teuthology.orchestra.run.smithi028.stdout:osd.6 smithi144 running (11m) 3m ago 11m 533M 2704M 17.2.0 e1d6a67b021e 7f9981be792f 2024-09-16T08:22:25.460 INFO:teuthology.orchestra.run.smithi028.stdout:osd.7 smithi144 running (11m) 3m ago 11m 429M 2704M 17.2.0 e1d6a67b021e dada74ddbe4c 2024-09-16T08:22:25.460 INFO:teuthology.orchestra.run.smithi028.stdout:prometheus.smithi028 smithi028 *:9095 running (5m) 13s ago 14m 55.3M - 2.51.0 1d3b7f56885b 42667809df2b 2024-09-16T08:22:25.985 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:22:25 smithi028 ceph-mon[64115]: from='client.34334 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T08:22:25.985 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:22:25 smithi028 ceph-mon[64115]: pgmap v185: 97 pgs: 1 peering, 3 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 349 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 5.0 KiB/s rd, 4.7 MiB/s wr, 556 op/s; 71/1629 objects degraded (4.359%); 236 KiB/s, 58 objects/s recovering 2024-09-16T08:22:25.985 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:22:25 smithi028 ceph-mon[64115]: from='client.34338 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T08:22:26.071 INFO:teuthology.orchestra.run.smithi028.stdout: 5 229751 9.77 MB/sec execute 444 sec latency 33.353 ms 2024-09-16T08:22:26.172 INFO:teuthology.orchestra.run.smithi028.stdout:{ 2024-09-16T08:22:26.172 INFO:teuthology.orchestra.run.smithi028.stdout: "mon": { 2024-09-16T08:22:26.172 INFO:teuthology.orchestra.run.smithi028.stdout: "ceph version 19.3.0-4917-g8293d73f (8293d73f8690540e843a81caec373f9cc29cf705) squid (dev)": 2 2024-09-16T08:22:26.172 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-09-16T08:22:26.172 INFO:teuthology.orchestra.run.smithi028.stdout: "mgr": { 2024-09-16T08:22:26.173 INFO:teuthology.orchestra.run.smithi028.stdout: "ceph version 19.3.0-4917-g8293d73f (8293d73f8690540e843a81caec373f9cc29cf705) squid (dev)": 2 2024-09-16T08:22:26.173 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-09-16T08:22:26.173 INFO:teuthology.orchestra.run.smithi028.stdout: "osd": { 2024-09-16T08:22:26.173 INFO:teuthology.orchestra.run.smithi028.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 4, 2024-09-16T08:22:26.173 INFO:teuthology.orchestra.run.smithi028.stdout: "ceph version 19.3.0-4917-g8293d73f (8293d73f8690540e843a81caec373f9cc29cf705) squid (dev)": 4 2024-09-16T08:22:26.173 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-09-16T08:22:26.173 INFO:teuthology.orchestra.run.smithi028.stdout: "mds": { 2024-09-16T08:22:26.173 INFO:teuthology.orchestra.run.smithi028.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-16T08:22:26.173 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-09-16T08:22:26.173 INFO:teuthology.orchestra.run.smithi028.stdout: "overall": { 2024-09-16T08:22:26.173 INFO:teuthology.orchestra.run.smithi028.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 6, 2024-09-16T08:22:26.173 INFO:teuthology.orchestra.run.smithi028.stdout: "ceph version 19.3.0-4917-g8293d73f (8293d73f8690540e843a81caec373f9cc29cf705) squid (dev)": 8 2024-09-16T08:22:26.174 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-09-16T08:22:26.174 INFO:teuthology.orchestra.run.smithi028.stdout:} 2024-09-16T08:22:26.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:22:25 smithi144 ceph-mon[49149]: from='client.34334 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T08:22:26.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:22:25 smithi144 ceph-mon[49149]: pgmap v185: 97 pgs: 1 peering, 3 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 349 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 5.0 KiB/s rd, 4.7 MiB/s wr, 556 op/s; 71/1629 objects degraded (4.359%); 236 KiB/s, 58 objects/s recovering 2024-09-16T08:22:26.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:22:25 smithi144 ceph-mon[49149]: from='client.34338 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T08:22:26.757 INFO:teuthology.orchestra.run.smithi028.stdout:{ 2024-09-16T08:22:26.757 INFO:teuthology.orchestra.run.smithi028.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:f7ad55f9ccd17ba66aadb19b4d291e67b60ea24e5d8a0c0e92941fd59c81e4ce", 2024-09-16T08:22:26.757 INFO:teuthology.orchestra.run.smithi028.stdout: "in_progress": true, 2024-09-16T08:22:26.757 INFO:teuthology.orchestra.run.smithi028.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-16T08:22:26.757 INFO:teuthology.orchestra.run.smithi028.stdout: "services_complete": [ 2024-09-16T08:22:26.757 INFO:teuthology.orchestra.run.smithi028.stdout: "crash", 2024-09-16T08:22:26.758 INFO:teuthology.orchestra.run.smithi028.stdout: "mon", 2024-09-16T08:22:26.758 INFO:teuthology.orchestra.run.smithi028.stdout: "mgr" 2024-09-16T08:22:26.758 INFO:teuthology.orchestra.run.smithi028.stdout: ], 2024-09-16T08:22:26.758 INFO:teuthology.orchestra.run.smithi028.stdout: "progress": "10/23 daemons upgraded", 2024-09-16T08:22:26.758 INFO:teuthology.orchestra.run.smithi028.stdout: "message": "Currently upgrading osd daemons", 2024-09-16T08:22:26.758 INFO:teuthology.orchestra.run.smithi028.stdout: "is_paused": false 2024-09-16T08:22:26.758 INFO:teuthology.orchestra.run.smithi028.stdout:} 2024-09-16T08:22:27.025 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:22:26 smithi028 ceph-mon[64115]: from='client.34342 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T08:22:27.025 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:22:26 smithi028 ceph-mon[64115]: from='client.? 172.21.15.28:0/3328356315' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-16T08:22:27.071 INFO:teuthology.orchestra.run.smithi028.stdout: 5 230010 9.77 MB/sec execute 445 sec latency 60.505 ms 2024-09-16T08:22:27.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:22:26 smithi144 ceph-mon[49149]: from='client.34342 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T08:22:27.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:22:26 smithi144 ceph-mon[49149]: from='client.? 172.21.15.28:0/3328356315' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-16T08:22:27.534 INFO:teuthology.orchestra.run.smithi028.stdout:HEALTH_OK 2024-09-16T08:22:28.040 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:22:27 smithi028 ceph-mon[64115]: from='client.34348 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T08:22:28.040 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:22:27 smithi028 ceph-mon[64115]: pgmap v186: 97 pgs: 1 peering, 96 active+clean; 349 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 3.4 KiB/s rd, 3.5 MiB/s wr, 391 op/s; 298 KiB/s, 51 objects/s recovering 2024-09-16T08:22:28.040 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:22:27 smithi028 ceph-mon[64115]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 71/1629 objects degraded (4.359%), 3 pgs degraded) 2024-09-16T08:22:28.040 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:22:27 smithi028 ceph-mon[64115]: Cluster is now healthy 2024-09-16T08:22:28.040 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:22:27 smithi028 ceph-mon[64115]: from='client.? 172.21.15.28:0/2384554219' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-16T08:22:28.071 INFO:teuthology.orchestra.run.smithi028.stdout: 5 230265 9.76 MB/sec execute 446 sec latency 48.273 ms 2024-09-16T08:22:28.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:22:27 smithi144 ceph-mon[49149]: from='client.34348 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T08:22:28.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:22:27 smithi144 ceph-mon[49149]: pgmap v186: 97 pgs: 1 peering, 96 active+clean; 349 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 3.4 KiB/s rd, 3.5 MiB/s wr, 391 op/s; 298 KiB/s, 51 objects/s recovering 2024-09-16T08:22:28.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:22:27 smithi144 ceph-mon[49149]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 71/1629 objects degraded (4.359%), 3 pgs degraded) 2024-09-16T08:22:28.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:22:27 smithi144 ceph-mon[49149]: Cluster is now healthy 2024-09-16T08:22:28.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:22:27 smithi144 ceph-mon[49149]: from='client.? 172.21.15.28:0/2384554219' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-16T08:22:29.071 INFO:teuthology.orchestra.run.smithi028.stdout: 5 230519 9.75 MB/sec execute 447 sec latency 63.198 ms 2024-09-16T08:22:30.070 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:22:29 smithi028 ceph-mon[64115]: pgmap v187: 97 pgs: 97 active+clean; 349 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 2.9 KiB/s rd, 4.7 MiB/s wr, 392 op/s; 255 KiB/s, 44 objects/s recovering 2024-09-16T08:22:30.071 INFO:teuthology.orchestra.run.smithi028.stdout: 5 230775 9.74 MB/sec execute 448 sec latency 80.074 ms 2024-09-16T08:22:30.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:22:29 smithi144 ceph-mon[49149]: pgmap v187: 97 pgs: 97 active+clean; 349 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 2.9 KiB/s rd, 4.7 MiB/s wr, 392 op/s; 255 KiB/s, 44 objects/s recovering 2024-09-16T08:22:31.071 INFO:teuthology.orchestra.run.smithi028.stdout: 5 231027 9.73 MB/sec execute 449 sec latency 76.422 ms 2024-09-16T08:22:32.071 INFO:teuthology.orchestra.run.smithi028.stdout: 5 231396 9.72 MB/sec execute 450 sec latency 66.271 ms 2024-09-16T08:22:32.114 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:22:31 smithi028 ceph-mon[64115]: pgmap v188: 97 pgs: 97 active+clean; 354 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 5.9 KiB/s rd, 4.5 MiB/s wr, 399 op/s; 213 KiB/s, 28 objects/s recovering 2024-09-16T08:22:32.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:22:31 smithi144 ceph-mon[49149]: pgmap v188: 97 pgs: 97 active+clean; 354 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 5.9 KiB/s rd, 4.5 MiB/s wr, 399 op/s; 213 KiB/s, 28 objects/s recovering 2024-09-16T08:22:33.071 INFO:teuthology.orchestra.run.smithi028.stdout: 5 231737 9.72 MB/sec execute 451 sec latency 63.021 ms 2024-09-16T08:22:33.401 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:22:33 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:22:33.401 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:22:33 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-16T08:22:33.401 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:22:33 smithi028 ceph-mon[64115]: pgmap v189: 97 pgs: 97 active+clean; 350 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 4.7 KiB/s rd, 4.2 MiB/s wr, 471 op/s; 171 KiB/s, 20 objects/s recovering 2024-09-16T08:22:33.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:22:33 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:22:33.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:22:33 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-16T08:22:33.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:22:33 smithi144 ceph-mon[49149]: pgmap v189: 97 pgs: 97 active+clean; 350 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 4.7 KiB/s rd, 4.2 MiB/s wr, 471 op/s; 171 KiB/s, 20 objects/s recovering 2024-09-16T08:22:34.071 INFO:teuthology.orchestra.run.smithi028.stdout: 5 232075 9.71 MB/sec execute 452 sec latency 53.181 ms 2024-09-16T08:22:34.924 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:22:34 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16} : dispatch 2024-09-16T08:22:35.071 INFO:teuthology.orchestra.run.smithi028.stdout: 5 232370 9.72 MB/sec execute 453 sec latency 53.388 ms 2024-09-16T08:22:35.185 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:22:34 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16} : dispatch 2024-09-16T08:22:35.947 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:22:35 smithi028 ceph-mon[64115]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-09-16T08:22:35.947 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:22:35 smithi028 ceph-mon[64115]: Upgrade: osd.4 is safe to restart 2024-09-16T08:22:35.948 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:22:35 smithi028 ceph-mon[64115]: pgmap v190: 97 pgs: 97 active+clean; 343 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 8.3 KiB/s rd, 4.3 MiB/s wr, 459 op/s; 156 KiB/s, 19 objects/s recovering 2024-09-16T08:22:36.073 INFO:teuthology.orchestra.run.smithi028.stdout: 5 232839 9.73 MB/sec execute 454 sec latency 37.739 ms 2024-09-16T08:22:36.311 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:22:35 smithi144 ceph-mon[49149]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-09-16T08:22:36.311 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:22:35 smithi144 ceph-mon[49149]: Upgrade: osd.4 is safe to restart 2024-09-16T08:22:36.311 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:22:35 smithi144 ceph-mon[49149]: pgmap v190: 97 pgs: 97 active+clean; 343 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 8.3 KiB/s rd, 4.3 MiB/s wr, 459 op/s; 156 KiB/s, 19 objects/s recovering 2024-09-16T08:22:37.072 INFO:teuthology.orchestra.run.smithi028.stdout: 5 233149 9.73 MB/sec execute 455 sec latency 53.331 ms 2024-09-16T08:22:37.456 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:22:37 smithi028 ceph-mon[64115]: Upgrade: Updating osd.4 2024-09-16T08:22:37.456 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:22:37 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:22:37.456 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:22:37 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "auth get", "entity": "osd.4"} : dispatch 2024-09-16T08:22:37.457 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:22:37 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-16T08:22:37.457 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:22:37 smithi028 ceph-mon[64115]: Deploying daemon osd.4 on smithi144 2024-09-16T08:22:37.457 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:22:37 smithi028 ceph-mon[64115]: pgmap v191: 97 pgs: 97 active+clean; 342 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 5.9 KiB/s rd, 3.4 MiB/s wr, 318 op/s; 69 KiB/s, 5 objects/s recovering 2024-09-16T08:22:37.764 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:22:37 smithi144 ceph-mon[49149]: Upgrade: Updating osd.4 2024-09-16T08:22:37.765 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:22:37 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:22:37.765 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:22:37 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "auth get", "entity": "osd.4"} : dispatch 2024-09-16T08:22:37.765 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:22:37 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-16T08:22:37.765 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:22:37 smithi144 ceph-mon[49149]: Deploying daemon osd.4 on smithi144 2024-09-16T08:22:37.765 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:22:37 smithi144 ceph-mon[49149]: pgmap v191: 97 pgs: 97 active+clean; 342 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 5.9 KiB/s rd, 3.4 MiB/s wr, 318 op/s; 69 KiB/s, 5 objects/s recovering 2024-09-16T08:22:38.071 INFO:teuthology.orchestra.run.smithi028.stdout: 5 233368 9.73 MB/sec execute 456 sec latency 72.273 ms 2024-09-16T08:22:38.522 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:22:38 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:22:38.648 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:22:38 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:22:39.072 INFO:teuthology.orchestra.run.smithi028.stdout: 5 233564 9.72 MB/sec execute 457 sec latency 135.927 ms 2024-09-16T08:22:39.559 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:22:39 smithi028 ceph-mon[64115]: pgmap v192: 97 pgs: 97 active+clean; 347 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 5.9 KiB/s rd, 4.9 MiB/s wr, 481 op/s 2024-09-16T08:22:39.740 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:22:39 smithi144 ceph-mon[49149]: pgmap v192: 97 pgs: 97 active+clean; 347 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 5.9 KiB/s rd, 4.9 MiB/s wr, 481 op/s 2024-09-16T08:22:40.023 INFO:journalctl@ceph.osd.4.smithi144.stdout:Sep 16 08:22:39 smithi144 systemd[1]: Stopping Ceph osd.4 for 5622b2fe-7402-11ef-bceb-c7b262605968... 2024-09-16T08:22:40.072 INFO:teuthology.orchestra.run.smithi028.stdout: 5 233783 9.71 MB/sec execute 458 sec latency 70.859 ms 2024-09-16T08:22:40.273 INFO:journalctl@ceph.osd.4.smithi144.stdout:Sep 16 08:22:40 smithi144 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-4[29728]: 2024-09-16T08:22:40.054+0000 7fd17eda4700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.4 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-16T08:22:40.273 INFO:journalctl@ceph.osd.4.smithi144.stdout:Sep 16 08:22:40 smithi144 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-4[29728]: 2024-09-16T08:22:40.054+0000 7fd17eda4700 -1 osd.4 115 *** Got signal Terminated *** 2024-09-16T08:22:40.273 INFO:journalctl@ceph.osd.4.smithi144.stdout:Sep 16 08:22:40 smithi144 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-4[29728]: 2024-09-16T08:22:40.054+0000 7fd17eda4700 -1 osd.4 115 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-16T08:22:40.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:22:40 smithi144 ceph-mon[49149]: osd.4 marked itself down and dead 2024-09-16T08:22:40.697 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:22:40 smithi028 ceph-mon[64115]: osd.4 marked itself down and dead 2024-09-16T08:22:41.072 INFO:teuthology.orchestra.run.smithi028.stdout: 5 233877 9.69 MB/sec execute 459 sec latency 645.013 ms 2024-09-16T08:22:41.332 INFO:journalctl@ceph.osd.4.smithi144.stdout:Sep 16 08:22:41 smithi144 podman[52565]: 2024-09-16 08:22:41.059827409 +0000 UTC m=+1.122534105 container died 86719c60975c7aada0611f147b94cc4233a5184c9cebaf1bac9c18bc02271dd8 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-4, build-date=2022-05-03T08:36:31.336870, GIT_BRANCH=HEAD, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., maintainer=Guillaume Abrioux , ceph=True, CEPH_POINT_RELEASE=-17.2.0, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_CLEAN=True, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.openshift.expose-services=, io.k8s.display-name=CentOS Stream 8, io.openshift.tags=base centos centos-stream, vcs-type=git, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, RELEASE=HEAD, version=8, GIT_REPO=https://github.com/ceph/ceph-container.git, vendor=Red Hat, Inc., release=754, name=centos-stream, io.buildah.version=1.19.8, architecture=x86_64, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, distribution-scope=public, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.component=centos-stream-container) 2024-09-16T08:22:41.684 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:22:41 smithi028 ceph-mon[64115]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-16T08:22:41.684 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:22:41 smithi028 ceph-mon[64115]: osdmap e116: 8 total, 7 up, 8 in 2024-09-16T08:22:41.684 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:22:41 smithi028 ceph-mon[64115]: pgmap v194: 97 pgs: 14 stale+active+clean, 83 active+clean; 350 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 6.9 KiB/s rd, 4.2 MiB/s wr, 446 op/s 2024-09-16T08:22:41.736 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:22:41 smithi144 ceph-mon[49149]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-16T08:22:41.736 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:22:41 smithi144 ceph-mon[49149]: osdmap e116: 8 total, 7 up, 8 in 2024-09-16T08:22:41.736 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:22:41 smithi144 ceph-mon[49149]: pgmap v194: 97 pgs: 14 stale+active+clean, 83 active+clean; 350 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 6.9 KiB/s rd, 4.2 MiB/s wr, 446 op/s 2024-09-16T08:22:41.736 INFO:journalctl@ceph.osd.4.smithi144.stdout:Sep 16 08:22:41 smithi144 podman[52565]: 2024-09-16 08:22:41.434805761 +0000 UTC m=+1.497512464 container cleanup 86719c60975c7aada0611f147b94cc4233a5184c9cebaf1bac9c18bc02271dd8 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-4, version=8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, build-date=2022-05-03T08:36:31.336870, vcs-type=git, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, CEPH_POINT_RELEASE=-17.2.0, vendor=Red Hat, Inc., RELEASE=HEAD, io.k8s.display-name=CentOS Stream 8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_CLEAN=True, distribution-scope=public, com.redhat.component=centos-stream-container, io.openshift.tags=base centos centos-stream, GIT_REPO=https://github.com/ceph/ceph-container.git, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, name=centos-stream, io.openshift.expose-services=, GIT_BRANCH=HEAD, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, ceph=True, io.buildah.version=1.19.8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., release=754, architecture=x86_64, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, maintainer=Guillaume Abrioux ) 2024-09-16T08:22:41.737 INFO:journalctl@ceph.osd.4.smithi144.stdout:Sep 16 08:22:41 smithi144 bash[52565]: ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-4 2024-09-16T08:22:42.072 INFO:teuthology.orchestra.run.smithi028.stdout: 5 234041 9.68 MB/sec execute 460 sec latency 1062.371 ms 2024-09-16T08:22:42.081 INFO:journalctl@ceph.osd.4.smithi144.stdout:Sep 16 08:22:41 smithi144 podman[52591]: 2024-09-16 08:22:41.736617663 +0000 UTC m=+0.674753319 container remove 86719c60975c7aada0611f147b94cc4233a5184c9cebaf1bac9c18bc02271dd8 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-4, GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_CLEAN=True, maintainer=Guillaume Abrioux , com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vcs-type=git, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_BRANCH=HEAD, release=754, com.redhat.component=centos-stream-container, name=centos-stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, RELEASE=HEAD, distribution-scope=public, io.openshift.tags=base centos centos-stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.k8s.display-name=CentOS Stream 8, vendor=Red Hat, Inc., build-date=2022-05-03T08:36:31.336870, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, architecture=x86_64, CEPH_POINT_RELEASE=-17.2.0, io.buildah.version=1.19.8, version=8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., ceph=True, io.openshift.expose-services=, com.redhat.license_terms=https://centos.org/legal/licensing-policy/) 2024-09-16T08:22:42.424 INFO:journalctl@ceph.osd.4.smithi144.stdout:Sep 16 08:22:42 smithi144 podman[52678]: 2024-09-16 08:22:42.038518821 +0000 UTC m=+0.017363914 image pull 425e316ba8ebf74f04aa3640f7deb4fa3f293a78d0c8d097cb679eaf86ddca6c quay.ceph.io/ceph-ci/ceph@sha256:f7ad55f9ccd17ba66aadb19b4d291e67b60ea24e5d8a0c0e92941fd59c81e4ce 2024-09-16T08:22:42.425 INFO:journalctl@ceph.osd.4.smithi144.stdout:Sep 16 08:22:42 smithi144 podman[52678]: 2024-09-16 08:22:42.162985802 +0000 UTC m=+0.141830869 container create 585265c29a4fe42c2b3edb90b270ffd5d3e087b6d7fc22ac483248b07c10913e (image=quay.ceph.io/ceph-ci/ceph@sha256:f7ad55f9ccd17ba66aadb19b4d291e67b60ea24e5d8a0c0e92941fd59c81e4ce, name=ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-4-deactivate, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, RELEASE=wip-guits-main-2024-09-13-1248-8293d73, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, ceph=True, io.buildah.version=1.37.2) 2024-09-16T08:22:42.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:22:42 smithi028 ceph-mon[64115]: osdmap e117: 8 total, 7 up, 8 in 2024-09-16T08:22:42.743 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:22:42 smithi144 ceph-mon[49149]: osdmap e117: 8 total, 7 up, 8 in 2024-09-16T08:22:42.744 INFO:journalctl@ceph.osd.4.smithi144.stdout:Sep 16 08:22:42 smithi144 podman[52678]: 2024-09-16 08:22:42.425103068 +0000 UTC m=+0.403948147 container init 585265c29a4fe42c2b3edb90b270ffd5d3e087b6d7fc22ac483248b07c10913e (image=quay.ceph.io/ceph-ci/ceph@sha256:f7ad55f9ccd17ba66aadb19b4d291e67b60ea24e5d8a0c0e92941fd59c81e4ce, name=ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-4-deactivate, CEPH_POINT_RELEASE=, RELEASE=wip-guits-main-2024-09-13-1248-8293d73, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, ceph=True, org.label-schema.build-date=20240731, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-16T08:22:42.744 INFO:journalctl@ceph.osd.4.smithi144.stdout:Sep 16 08:22:42 smithi144 podman[52678]: 2024-09-16 08:22:42.429852595 +0000 UTC m=+0.408697674 container start 585265c29a4fe42c2b3edb90b270ffd5d3e087b6d7fc22ac483248b07c10913e (image=quay.ceph.io/ceph-ci/ceph@sha256:f7ad55f9ccd17ba66aadb19b4d291e67b60ea24e5d8a0c0e92941fd59c81e4ce, name=ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-4-deactivate, maintainer=Guillaume Abrioux , RELEASE=wip-guits-main-2024-09-13-1248-8293d73, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_CLEAN=True, ceph=True, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0) 2024-09-16T08:22:42.744 INFO:journalctl@ceph.osd.4.smithi144.stdout:Sep 16 08:22:42 smithi144 podman[52678]: 2024-09-16 08:22:42.74404789 +0000 UTC m=+0.722892983 container attach 585265c29a4fe42c2b3edb90b270ffd5d3e087b6d7fc22ac483248b07c10913e (image=quay.ceph.io/ceph-ci/ceph@sha256:f7ad55f9ccd17ba66aadb19b4d291e67b60ea24e5d8a0c0e92941fd59c81e4ce, name=ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-4-deactivate, GIT_CLEAN=True, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.schema-version=1.0, RELEASE=wip-guits-main-2024-09-13-1248-8293d73, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS) 2024-09-16T08:22:43.072 INFO:teuthology.orchestra.run.smithi028.stdout: 5 234416 9.68 MB/sec execute 461 sec latency 31.328 ms 2024-09-16T08:22:43.082 INFO:journalctl@ceph.osd.4.smithi144.stdout:Sep 16 08:22:42 smithi144 podman[52678]: 2024-09-16 08:22:42.744351234 +0000 UTC m=+0.723196320 container died 585265c29a4fe42c2b3edb90b270ffd5d3e087b6d7fc22ac483248b07c10913e (image=quay.ceph.io/ceph-ci/ceph@sha256:f7ad55f9ccd17ba66aadb19b4d291e67b60ea24e5d8a0c0e92941fd59c81e4ce, name=ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-4-deactivate, RELEASE=wip-guits-main-2024-09-13-1248-8293d73, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, ceph=True, org.label-schema.license=GPLv2) 2024-09-16T08:22:43.453 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:22:43 smithi028 ceph-mon[64115]: pgmap v196: 97 pgs: 6 active+undersized, 8 stale+active+clean, 11 active+undersized+degraded, 72 active+clean; 348 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 3.1 KiB/s rd, 4.3 MiB/s wr, 385 op/s; 82/1689 objects degraded (4.855%) 2024-09-16T08:22:43.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:22:43 smithi144 ceph-mon[49149]: pgmap v196: 97 pgs: 6 active+undersized, 8 stale+active+clean, 11 active+undersized+degraded, 72 active+clean; 348 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 3.1 KiB/s rd, 4.3 MiB/s wr, 385 op/s; 82/1689 objects degraded (4.855%) 2024-09-16T08:22:43.582 INFO:journalctl@ceph.osd.4.smithi144.stdout:Sep 16 08:22:43 smithi144 podman[52701]: 2024-09-16 08:22:43.447165318 +0000 UTC m=+0.770515546 container remove 585265c29a4fe42c2b3edb90b270ffd5d3e087b6d7fc22ac483248b07c10913e (image=quay.ceph.io/ceph-ci/ceph@sha256:f7ad55f9ccd17ba66aadb19b4d291e67b60ea24e5d8a0c0e92941fd59c81e4ce, name=ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-4-deactivate, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, RELEASE=wip-guits-main-2024-09-13-1248-8293d73, org.label-schema.license=GPLv2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_CLEAN=True, ceph=True, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0) 2024-09-16T08:22:43.582 INFO:journalctl@ceph.osd.4.smithi144.stdout:Sep 16 08:22:43 smithi144 systemd[1]: ceph-5622b2fe-7402-11ef-bceb-c7b262605968@osd.4.service: Deactivated successfully. 2024-09-16T08:22:43.582 INFO:journalctl@ceph.osd.4.smithi144.stdout:Sep 16 08:22:43 smithi144 systemd[1]: Stopped Ceph osd.4 for 5622b2fe-7402-11ef-bceb-c7b262605968. 2024-09-16T08:22:43.582 INFO:journalctl@ceph.osd.4.smithi144.stdout:Sep 16 08:22:43 smithi144 systemd[1]: ceph-5622b2fe-7402-11ef-bceb-c7b262605968@osd.4.service: Consumed 4min 1.936s CPU time. 2024-09-16T08:22:43.973 INFO:journalctl@ceph.osd.4.smithi144.stdout:Sep 16 08:22:43 smithi144 systemd[1]: Starting Ceph osd.4 for 5622b2fe-7402-11ef-bceb-c7b262605968... 2024-09-16T08:22:43.973 INFO:journalctl@ceph.osd.4.smithi144.stdout:Sep 16 08:22:43 smithi144 podman[52798]: 2024-09-16 08:22:43.83901088 +0000 UTC m=+0.018943285 image pull 425e316ba8ebf74f04aa3640f7deb4fa3f293a78d0c8d097cb679eaf86ddca6c quay.ceph.io/ceph-ci/ceph@sha256:f7ad55f9ccd17ba66aadb19b4d291e67b60ea24e5d8a0c0e92941fd59c81e4ce 2024-09-16T08:22:44.072 INFO:teuthology.orchestra.run.smithi028.stdout: 5 234850 9.67 MB/sec execute 462 sec latency 42.687 ms 2024-09-16T08:22:44.302 INFO:journalctl@ceph.osd.4.smithi144.stdout:Sep 16 08:22:43 smithi144 podman[52798]: 2024-09-16 08:22:43.97394804 +0000 UTC m=+0.153880427 container create b0ac387d1f3567e079fddbafbdefe8df706974c2e8c58575da59b8ec588a40bc (image=quay.ceph.io/ceph-ci/ceph@sha256:f7ad55f9ccd17ba66aadb19b4d291e67b60ea24e5d8a0c0e92941fd59c81e4ce, name=ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-4-activate, org.label-schema.vendor=CentOS, ceph=True, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, RELEASE=wip-guits-main-2024-09-13-1248-8293d73, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2) 2024-09-16T08:22:44.302 INFO:journalctl@ceph.osd.4.smithi144.stdout:Sep 16 08:22:44 smithi144 podman[52798]: 2024-09-16 08:22:44.219237943 +0000 UTC m=+0.399170332 container init b0ac387d1f3567e079fddbafbdefe8df706974c2e8c58575da59b8ec588a40bc (image=quay.ceph.io/ceph-ci/ceph@sha256:f7ad55f9ccd17ba66aadb19b4d291e67b60ea24e5d8a0c0e92941fd59c81e4ce, name=ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-4-activate, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, ceph=True, GIT_BRANCH=HEAD, RELEASE=wip-guits-main-2024-09-13-1248-8293d73, io.buildah.version=1.37.2, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2) 2024-09-16T08:22:44.302 INFO:journalctl@ceph.osd.4.smithi144.stdout:Sep 16 08:22:44 smithi144 podman[52798]: 2024-09-16 08:22:44.223642742 +0000 UTC m=+0.403575126 container start b0ac387d1f3567e079fddbafbdefe8df706974c2e8c58575da59b8ec588a40bc (image=quay.ceph.io/ceph-ci/ceph@sha256:f7ad55f9ccd17ba66aadb19b4d291e67b60ea24e5d8a0c0e92941fd59c81e4ce, name=ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-4-activate, org.label-schema.build-date=20240731, ceph=True, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-guits-main-2024-09-13-1248-8293d73, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-16T08:22:44.302 INFO:journalctl@ceph.osd.4.smithi144.stdout:Sep 16 08:22:44 smithi144 podman[52798]: 2024-09-16 08:22:44.302558916 +0000 UTC m=+0.482491296 container attach b0ac387d1f3567e079fddbafbdefe8df706974c2e8c58575da59b8ec588a40bc (image=quay.ceph.io/ceph-ci/ceph@sha256:f7ad55f9ccd17ba66aadb19b4d291e67b60ea24e5d8a0c0e92941fd59c81e4ce, name=ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-4-activate, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , ceph=True, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, RELEASE=wip-guits-main-2024-09-13-1248-8293d73, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965) 2024-09-16T08:22:44.473 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:22:44 smithi028 ceph-mon[64115]: Health check failed: Degraded data redundancy: 82/1689 objects degraded (4.855%), 11 pgs degraded (PG_DEGRADED) 2024-09-16T08:22:44.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:22:44 smithi144 ceph-mon[49149]: Health check failed: Degraded data redundancy: 82/1689 objects degraded (4.855%), 11 pgs degraded (PG_DEGRADED) 2024-09-16T08:22:44.582 INFO:journalctl@ceph.osd.4.smithi144.stdout:Sep 16 08:22:44 smithi144 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-4-activate[52813]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-16T08:22:44.582 INFO:journalctl@ceph.osd.4.smithi144.stdout:Sep 16 08:22:44 smithi144 bash[52798]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-16T08:22:44.582 INFO:journalctl@ceph.osd.4.smithi144.stdout:Sep 16 08:22:44 smithi144 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-4-activate[52813]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-16T08:22:44.583 INFO:journalctl@ceph.osd.4.smithi144.stdout:Sep 16 08:22:44 smithi144 bash[52798]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-16T08:22:45.072 INFO:teuthology.orchestra.run.smithi028.stdout: 5 235224 9.67 MB/sec execute 463 sec latency 72.338 ms 2024-09-16T08:22:45.488 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:22:45 smithi028 ceph-mon[64115]: pgmap v197: 97 pgs: 12 active+undersized, 2 stale+active+clean, 21 active+undersized+degraded, 62 active+clean; 351 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 7.2 KiB/s rd, 5.1 MiB/s wr, 485 op/s; 198/1659 objects degraded (11.935%) 2024-09-16T08:22:45.831 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:22:45 smithi144 ceph-mon[49149]: pgmap v197: 97 pgs: 12 active+undersized, 2 stale+active+clean, 21 active+undersized+degraded, 62 active+clean; 351 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 7.2 KiB/s rd, 5.1 MiB/s wr, 485 op/s; 198/1659 objects degraded (11.935%) 2024-09-16T08:22:46.072 INFO:teuthology.orchestra.run.smithi028.stdout: 5 235455 9.65 MB/sec execute 464 sec latency 81.570 ms 2024-09-16T08:22:47.037 INFO:journalctl@ceph.osd.4.smithi144.stdout:Sep 16 08:22:46 smithi144 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-4-activate[52813]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2024-09-16T08:22:47.037 INFO:journalctl@ceph.osd.4.smithi144.stdout:Sep 16 08:22:46 smithi144 bash[52798]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2024-09-16T08:22:47.038 INFO:journalctl@ceph.osd.4.smithi144.stdout:Sep 16 08:22:46 smithi144 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-4-activate[52813]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-4 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2024-09-16T08:22:47.038 INFO:journalctl@ceph.osd.4.smithi144.stdout:Sep 16 08:22:46 smithi144 bash[52798]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-4 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2024-09-16T08:22:47.038 INFO:journalctl@ceph.osd.4.smithi144.stdout:Sep 16 08:22:47 smithi144 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-4-activate[52813]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2024-09-16T08:22:47.038 INFO:journalctl@ceph.osd.4.smithi144.stdout:Sep 16 08:22:47 smithi144 bash[52798]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2024-09-16T08:22:47.072 INFO:teuthology.orchestra.run.smithi028.stdout: 5 235770 9.66 MB/sec execute 465 sec latency 60.341 ms 2024-09-16T08:22:47.332 INFO:journalctl@ceph.osd.4.smithi144.stdout:Sep 16 08:22:47 smithi144 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-4-activate[52813]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-09-16T08:22:47.332 INFO:journalctl@ceph.osd.4.smithi144.stdout:Sep 16 08:22:47 smithi144 bash[52798]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-09-16T08:22:47.332 INFO:journalctl@ceph.osd.4.smithi144.stdout:Sep 16 08:22:47 smithi144 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-4-activate[52813]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-4/block 2024-09-16T08:22:47.332 INFO:journalctl@ceph.osd.4.smithi144.stdout:Sep 16 08:22:47 smithi144 bash[52798]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-4/block 2024-09-16T08:22:47.332 INFO:journalctl@ceph.osd.4.smithi144.stdout:Sep 16 08:22:47 smithi144 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-4-activate[52813]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2024-09-16T08:22:47.332 INFO:journalctl@ceph.osd.4.smithi144.stdout:Sep 16 08:22:47 smithi144 bash[52798]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2024-09-16T08:22:47.332 INFO:journalctl@ceph.osd.4.smithi144.stdout:Sep 16 08:22:47 smithi144 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-4-activate[52813]: --> ceph-volume raw activate successful for osd ID: 4 2024-09-16T08:22:47.332 INFO:journalctl@ceph.osd.4.smithi144.stdout:Sep 16 08:22:47 smithi144 bash[52798]: --> ceph-volume raw activate successful for osd ID: 4 2024-09-16T08:22:47.332 INFO:journalctl@ceph.osd.4.smithi144.stdout:Sep 16 08:22:47 smithi144 podman[52798]: 2024-09-16 08:22:47.073411987 +0000 UTC m=+3.253344375 container died b0ac387d1f3567e079fddbafbdefe8df706974c2e8c58575da59b8ec588a40bc (image=quay.ceph.io/ceph-ci/ceph@sha256:f7ad55f9ccd17ba66aadb19b4d291e67b60ea24e5d8a0c0e92941fd59c81e4ce, name=ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-4-activate, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, ceph=True, CEPH_POINT_RELEASE=, RELEASE=wip-guits-main-2024-09-13-1248-8293d73, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, GIT_CLEAN=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965) 2024-09-16T08:22:47.958 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:22:47 smithi144 ceph-mon[49149]: pgmap v198: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 348 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 7.2 KiB/s rd, 3.0 MiB/s wr, 255 op/s; 263/1635 objects degraded (16.086%) 2024-09-16T08:22:47.958 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:22:47 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:22:47.958 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:22:47 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-16T08:22:47.959 INFO:journalctl@ceph.osd.4.smithi144.stdout:Sep 16 08:22:47 smithi144 podman[52965]: 2024-09-16 08:22:47.698953646 +0000 UTC m=+0.614038614 container remove b0ac387d1f3567e079fddbafbdefe8df706974c2e8c58575da59b8ec588a40bc (image=quay.ceph.io/ceph-ci/ceph@sha256:f7ad55f9ccd17ba66aadb19b4d291e67b60ea24e5d8a0c0e92941fd59c81e4ce, name=ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-4-activate, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , RELEASE=wip-guits-main-2024-09-13-1248-8293d73, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_CLEAN=True, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-16T08:22:47.959 INFO:journalctl@ceph.osd.4.smithi144.stdout:Sep 16 08:22:47 smithi144 podman[53020]: 2024-09-16 08:22:47.822845727 +0000 UTC m=+0.016922595 image pull 425e316ba8ebf74f04aa3640f7deb4fa3f293a78d0c8d097cb679eaf86ddca6c quay.ceph.io/ceph-ci/ceph@sha256:f7ad55f9ccd17ba66aadb19b4d291e67b60ea24e5d8a0c0e92941fd59c81e4ce 2024-09-16T08:22:47.987 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:22:47 smithi028 ceph-mon[64115]: pgmap v198: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 348 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 7.2 KiB/s rd, 3.0 MiB/s wr, 255 op/s; 263/1635 objects degraded (16.086%) 2024-09-16T08:22:47.987 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:22:47 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:22:47.987 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:22:47 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-16T08:22:48.072 INFO:teuthology.orchestra.run.smithi028.stdout: 5 236017 9.65 MB/sec execute 466 sec latency 69.501 ms 2024-09-16T08:22:48.278 INFO:journalctl@ceph.osd.4.smithi144.stdout:Sep 16 08:22:47 smithi144 podman[53020]: 2024-09-16 08:22:47.95838545 +0000 UTC m=+0.152462315 container create c48cb68e901e50e238853991ed735fc87807751260f575193166aead0e2ac74f (image=quay.ceph.io/ceph-ci/ceph@sha256:f7ad55f9ccd17ba66aadb19b4d291e67b60ea24e5d8a0c0e92941fd59c81e4ce, name=ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-4, ceph=True, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, RELEASE=wip-guits-main-2024-09-13-1248-8293d73, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_BRANCH=HEAD) 2024-09-16T08:22:48.582 INFO:journalctl@ceph.osd.4.smithi144.stdout:Sep 16 08:22:48 smithi144 podman[53020]: 2024-09-16 08:22:48.278623578 +0000 UTC m=+0.472700433 container init c48cb68e901e50e238853991ed735fc87807751260f575193166aead0e2ac74f (image=quay.ceph.io/ceph-ci/ceph@sha256:f7ad55f9ccd17ba66aadb19b4d291e67b60ea24e5d8a0c0e92941fd59c81e4ce, name=ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-4, org.label-schema.license=GPLv2, ceph=True, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=wip-guits-main-2024-09-13-1248-8293d73) 2024-09-16T08:22:48.582 INFO:journalctl@ceph.osd.4.smithi144.stdout:Sep 16 08:22:48 smithi144 podman[53020]: 2024-09-16 08:22:48.282044358 +0000 UTC m=+0.476121220 container start c48cb68e901e50e238853991ed735fc87807751260f575193166aead0e2ac74f (image=quay.ceph.io/ceph-ci/ceph@sha256:f7ad55f9ccd17ba66aadb19b4d291e67b60ea24e5d8a0c0e92941fd59c81e4ce, name=ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-4, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, ceph=True, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, RELEASE=wip-guits-main-2024-09-13-1248-8293d73) 2024-09-16T08:22:48.582 INFO:journalctl@ceph.osd.4.smithi144.stdout:Sep 16 08:22:48 smithi144 ceph-osd[53039]: -- 172.21.15.144:0/217601365 <== mon.1 v2:172.21.15.144:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x556d9a29e1e0 con 0x556d994a9c00 2024-09-16T08:22:48.582 INFO:journalctl@ceph.osd.4.smithi144.stdout:Sep 16 08:22:48 smithi144 bash[53020]: c48cb68e901e50e238853991ed735fc87807751260f575193166aead0e2ac74f 2024-09-16T08:22:48.582 INFO:journalctl@ceph.osd.4.smithi144.stdout:Sep 16 08:22:48 smithi144 systemd[1]: Started Ceph osd.4 for 5622b2fe-7402-11ef-bceb-c7b262605968. 2024-09-16T08:22:49.072 INFO:teuthology.orchestra.run.smithi028.stdout: 5 236521 9.67 MB/sec execute 467 sec latency 60.655 ms 2024-09-16T08:22:49.495 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:22:49 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:22:49.495 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:22:49 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:22:49.495 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:22:49 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-16T08:22:49.496 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:22:49 smithi028 ceph-mon[64115]: pgmap v199: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 343 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 3.9 KiB/s rd, 3.9 MiB/s wr, 452 op/s; 259/1617 objects degraded (16.017%) 2024-09-16T08:22:49.652 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:22:49 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:22:49.652 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:22:49 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:22:49.652 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:22:49 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-16T08:22:49.652 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:22:49 smithi144 ceph-mon[49149]: pgmap v199: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 343 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 3.9 KiB/s rd, 3.9 MiB/s wr, 452 op/s; 259/1617 objects degraded (16.017%) 2024-09-16T08:22:49.972 INFO:journalctl@ceph.osd.4.smithi144.stdout:Sep 16 08:22:49 smithi144 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-4[53035]: 2024-09-16T08:22:49.650+0000 7f305cd82640 -1 Falling back to public interface 2024-09-16T08:22:50.072 INFO:teuthology.orchestra.run.smithi028.stdout: 5 236914 9.68 MB/sec execute 468 sec latency 81.253 ms 2024-09-16T08:22:50.507 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:22:50 smithi028 ceph-mon[64115]: Health check update: Degraded data redundancy: 259/1617 objects degraded (16.017%), 28 pgs degraded (PG_DEGRADED) 2024-09-16T08:22:50.554 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:22:50 smithi144 ceph-mon[49149]: Health check update: Degraded data redundancy: 259/1617 objects degraded (16.017%), 28 pgs degraded (PG_DEGRADED) 2024-09-16T08:22:51.072 INFO:teuthology.orchestra.run.smithi028.stdout: 5 237265 9.68 MB/sec execute 469 sec latency 35.391 ms 2024-09-16T08:22:51.520 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:22:51 smithi028 ceph-mon[64115]: pgmap v200: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 346 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 6.9 KiB/s rd, 4.3 MiB/s wr, 461 op/s; 255/1644 objects degraded (15.511%) 2024-09-16T08:22:51.781 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:22:51 smithi144 ceph-mon[49149]: pgmap v200: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 346 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 6.9 KiB/s rd, 4.3 MiB/s wr, 461 op/s; 255/1644 objects degraded (15.511%) 2024-09-16T08:22:52.072 INFO:teuthology.orchestra.run.smithi028.stdout: 5 237657 9.68 MB/sec execute 470 sec latency 60.215 ms 2024-09-16T08:22:53.072 INFO:teuthology.orchestra.run.smithi028.stdout: 5 237964 9.67 MB/sec execute 471 sec latency 33.127 ms 2024-09-16T08:22:53.303 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:22:53 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:22:53.303 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:22:53 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:22:53.303 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:22:53 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:22:53.303 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:22:53 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:22:53.376 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:22:53 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:22:53.376 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:22:53 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:22:53.376 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:22:53 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:22:53.376 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:22:53 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:22:54.081 INFO:teuthology.orchestra.run.smithi028.stdout: 5 238266 9.67 MB/sec execute 472 sec latency 54.331 ms 2024-09-16T08:22:54.311 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:22:54 smithi028 ceph-mon[64115]: pgmap v201: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 349 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 4.8 MiB/s wr, 499 op/s; 264/1725 objects degraded (15.304%) 2024-09-16T08:22:54.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:22:54 smithi144 ceph-mon[49149]: pgmap v201: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 349 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 4.8 MiB/s wr, 499 op/s; 264/1725 objects degraded (15.304%) 2024-09-16T08:22:55.081 INFO:teuthology.orchestra.run.smithi028.stdout: 5 238659 9.66 MB/sec execute 473 sec latency 63.945 ms 2024-09-16T08:22:55.257 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:22:55 smithi144 ceph-mon[49149]: Health check update: Degraded data redundancy: 264/1725 objects degraded (15.304%), 28 pgs degraded (PG_DEGRADED) 2024-09-16T08:22:55.328 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:22:55 smithi028 ceph-mon[64115]: Health check update: Degraded data redundancy: 264/1725 objects degraded (15.304%), 28 pgs degraded (PG_DEGRADED) 2024-09-16T08:22:56.081 INFO:teuthology.orchestra.run.smithi028.stdout: 5 238954 9.65 MB/sec execute 474 sec latency 48.446 ms 2024-09-16T08:22:56.347 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:22:56 smithi028 ceph-mon[64115]: pgmap v202: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 351 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 9.5 KiB/s rd, 5.2 MiB/s wr, 557 op/s; 275/1779 objects degraded (15.458%) 2024-09-16T08:22:56.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:22:56 smithi144 ceph-mon[49149]: pgmap v202: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 351 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 9.5 KiB/s rd, 5.2 MiB/s wr, 557 op/s; 275/1779 objects degraded (15.458%) 2024-09-16T08:22:57.081 INFO:teuthology.orchestra.run.smithi028.stdout: 5 239266 9.65 MB/sec execute 475 sec latency 43.981 ms 2024-09-16T08:22:57.081 INFO:journalctl@ceph.osd.4.smithi144.stdout:Sep 16 08:22:56 smithi144 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-4[53035]: 2024-09-16T08:22:56.742+0000 7f305cd82640 -1 osd.4 0 read_superblock omap replica is missing. 2024-09-16T08:22:57.893 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:22:57 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:22:57.893 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:22:57 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:22:57.893 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:22:57 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-16T08:22:57.893 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:22:57 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-16T08:22:57.894 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:22:57 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:22:57.894 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:22:57 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:22:57.894 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:22:57 smithi028 ceph-mon[64115]: pgmap v203: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 348 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.7 MiB/s wr, 491 op/s; 266/1725 objects degraded (15.420%) 2024-09-16T08:22:57.894 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:22:57 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-16T08:22:57.894 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:22:57 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "versions"} : dispatch 2024-09-16T08:22:57.894 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:22:57 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "versions"} : dispatch 2024-09-16T08:22:57.894 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:22:57 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "versions"} : dispatch 2024-09-16T08:22:57.894 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:22:57 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16} : dispatch 2024-09-16T08:22:58.081 INFO:teuthology.orchestra.run.smithi028.stdout: 5 239619 9.66 MB/sec execute 476 sec latency 41.542 ms 2024-09-16T08:22:58.082 INFO:journalctl@ceph.osd.4.smithi144.stdout:Sep 16 08:22:57 smithi144 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-4[53035]: 2024-09-16T08:22:57.925+0000 7f305cd82640 -1 osd.4 115 log_to_monitors true 2024-09-16T08:22:58.082 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:22:57 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:22:58.082 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:22:57 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:22:58.082 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:22:57 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-16T08:22:58.082 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:22:57 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-16T08:22:58.082 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:22:57 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:22:58.082 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:22:57 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:22:58.082 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:22:57 smithi144 ceph-mon[49149]: pgmap v203: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 348 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.7 MiB/s wr, 491 op/s; 266/1725 objects degraded (15.420%) 2024-09-16T08:22:58.082 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:22:57 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-16T08:22:58.082 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:22:57 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "versions"} : dispatch 2024-09-16T08:22:58.083 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:22:57 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "versions"} : dispatch 2024-09-16T08:22:58.083 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:22:57 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "versions"} : dispatch 2024-09-16T08:22:58.083 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:22:57 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16} : dispatch 2024-09-16T08:22:58.127 INFO:teuthology.orchestra.run.smithi028.stdout:true 2024-09-16T08:22:58.925 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:22:58 smithi028 ceph-mon[64115]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-09-16T08:22:58.925 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:22:58 smithi028 ceph-mon[64115]: Upgrade: unsafe to stop osd(s) at this time (6 PGs are or would become offline) 2024-09-16T08:22:58.925 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:22:58 smithi028 ceph-mon[64115]: from='osd.4 [v2:172.21.15.144:6800/1573842275,v1:172.21.15.144:6801/1573842275]' entity='osd.4' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]} : dispatch 2024-09-16T08:22:58.925 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:22:58 smithi028 ceph-mon[64115]: from='osd.4 ' entity='osd.4' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]} : dispatch 2024-09-16T08:22:58.925 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:22:58 smithi028 ceph-mon[64115]: from='client.44291 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T08:22:59.082 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:22:58 smithi144 ceph-mon[49149]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-09-16T08:22:59.082 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:22:58 smithi144 ceph-mon[49149]: Upgrade: unsafe to stop osd(s) at this time (6 PGs are or would become offline) 2024-09-16T08:22:59.082 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:22:58 smithi144 ceph-mon[49149]: from='osd.4 [v2:172.21.15.144:6800/1573842275,v1:172.21.15.144:6801/1573842275]' entity='osd.4' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]} : dispatch 2024-09-16T08:22:59.082 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:22:58 smithi144 ceph-mon[49149]: from='osd.4 ' entity='osd.4' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]} : dispatch 2024-09-16T08:22:59.082 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:22:58 smithi144 ceph-mon[49149]: from='client.44291 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T08:22:59.086 INFO:teuthology.orchestra.run.smithi028.stdout: 5 239977 9.66 MB/sec execute 477 sec latency 104.347 ms 2024-09-16T08:22:59.244 INFO:teuthology.orchestra.run.smithi028.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-16T08:22:59.244 INFO:teuthology.orchestra.run.smithi028.stdout:alertmanager.smithi028 smithi028 *:9093,9094 running (7m) 47s ago 16m 18.6M - 0.27.0 11f11916f8cd a14fcb4cb2b6 2024-09-16T08:22:59.244 INFO:teuthology.orchestra.run.smithi028.stdout:crash.smithi028 smithi028 running (4m) 47s ago 16m 6899k - 19.3.0-4917-g8293d73f 425e316ba8eb 4bb89df2a867 2024-09-16T08:22:59.244 INFO:teuthology.orchestra.run.smithi028.stdout:crash.smithi144 smithi144 running (3m) 7s ago 15m 6895k - 19.3.0-4917-g8293d73f 425e316ba8eb 24a96875d7af 2024-09-16T08:22:59.244 INFO:teuthology.orchestra.run.smithi028.stdout:grafana.smithi028 smithi028 *:3000 running (7m) 47s ago 16m 45.9M - dad864ee21e9 b55ba8b944a1 2024-09-16T08:22:59.244 INFO:teuthology.orchestra.run.smithi028.stdout:mds.foofs.smithi028.srtgev smithi028 running (10m) 47s ago 10m 38.5M - 17.2.0 e1d6a67b021e 250e7a7f754c 2024-09-16T08:22:59.244 INFO:teuthology.orchestra.run.smithi028.stdout:mds.foofs.smithi144.oozgsi smithi144 running (10m) 7s ago 10m 17.3M - 17.2.0 e1d6a67b021e ecf4c2bb0fdc 2024-09-16T08:22:59.245 INFO:teuthology.orchestra.run.smithi028.stdout:mgr.smithi028.azvyzy smithi028 *:8443,9283,8765 running (6m) 47s ago 17m 519M - 19.3.0-4917-g8293d73f 425e316ba8eb e01d4382d03f 2024-09-16T08:22:59.245 INFO:teuthology.orchestra.run.smithi028.stdout:mgr.smithi144.utzszj smithi144 *:8443,9283,8765 running (5m) 7s ago 15m 457M - 19.3.0-4917-g8293d73f 425e316ba8eb 71f7f802bffb 2024-09-16T08:22:59.245 INFO:teuthology.orchestra.run.smithi028.stdout:mon.smithi028 smithi028 running (5m) 47s ago 17m 49.8M 2048M 19.3.0-4917-g8293d73f 425e316ba8eb 14efcedc0088 2024-09-16T08:22:59.245 INFO:teuthology.orchestra.run.smithi028.stdout:mon.smithi144 smithi144 running (4m) 7s ago 15m 43.4M 2048M 19.3.0-4917-g8293d73f 425e316ba8eb c081c5ed44b5 2024-09-16T08:22:59.245 INFO:teuthology.orchestra.run.smithi028.stdout:nfs.foo.0.0.smithi028.dkttgj smithi028 *:2049 running (10m) 47s ago 10m 285M - 4.0 e1d6a67b021e b5dccc45a6ed 2024-09-16T08:22:59.245 INFO:teuthology.orchestra.run.smithi028.stdout:nfs.foo.1.0.smithi144.epipkg smithi144 *:2049 running (10m) 7s ago 10m 87.3M - 4.0 e1d6a67b021e faa89e61f50e 2024-09-16T08:22:59.245 INFO:teuthology.orchestra.run.smithi028.stdout:node-exporter.smithi028 smithi028 *:9100 running (7m) 47s ago 16m 9.92M - 1.7.0 72c9c2088986 9e2b8e8a63a9 2024-09-16T08:22:59.245 INFO:teuthology.orchestra.run.smithi028.stdout:node-exporter.smithi144 smithi144 *:9100 running (6m) 7s ago 15m 10.6M - 1.7.0 72c9c2088986 78834c5e992f 2024-09-16T08:22:59.245 INFO:teuthology.orchestra.run.smithi028.stdout:osd.0 smithi028 running (3m) 47s ago 14m 354M 1937M 19.3.0-4917-g8293d73f 425e316ba8eb 2fc02ba4f14f 2024-09-16T08:22:59.245 INFO:teuthology.orchestra.run.smithi028.stdout:osd.1 smithi028 running (2m) 47s ago 14m 266M 1937M 19.3.0-4917-g8293d73f 425e316ba8eb df34bd603d8f 2024-09-16T08:22:59.245 INFO:teuthology.orchestra.run.smithi028.stdout:osd.2 smithi028 running (106s) 47s ago 14m 220M 1937M 19.3.0-4917-g8293d73f 425e316ba8eb c92be6327b12 2024-09-16T08:22:59.245 INFO:teuthology.orchestra.run.smithi028.stdout:osd.3 smithi028 running (56s) 47s ago 13m 11.2M 1937M 19.3.0-4917-g8293d73f 425e316ba8eb bd84ba1224f2 2024-09-16T08:22:59.245 INFO:teuthology.orchestra.run.smithi028.stdout:osd.4 smithi144 running (11s) 7s ago 13m 13.3M 2704M 19.3.0-4917-g8293d73f 425e316ba8eb c48cb68e901e 2024-09-16T08:22:59.245 INFO:teuthology.orchestra.run.smithi028.stdout:osd.5 smithi144 running (12m) 7s ago 12m 556M 2704M 17.2.0 e1d6a67b021e 70452c145c60 2024-09-16T08:22:59.246 INFO:teuthology.orchestra.run.smithi028.stdout:osd.6 smithi144 running (12m) 7s ago 12m 478M 2704M 17.2.0 e1d6a67b021e 7f9981be792f 2024-09-16T08:22:59.246 INFO:teuthology.orchestra.run.smithi028.stdout:osd.7 smithi144 running (12m) 7s ago 12m 725M 2704M 17.2.0 e1d6a67b021e dada74ddbe4c 2024-09-16T08:22:59.246 INFO:teuthology.orchestra.run.smithi028.stdout:prometheus.smithi028 smithi028 *:9095 running (5m) 47s ago 15m 55.3M - 2.51.0 1d3b7f56885b 42667809df2b 2024-09-16T08:22:59.582 INFO:journalctl@ceph.osd.4.smithi144.stdout:Sep 16 08:22:59 smithi144 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-4[53035]: 2024-09-16T08:22:59.180+0000 7f3050b24640 -1 osd.4 115 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-16T08:22:59.940 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:22:59 smithi028 ceph-mon[64115]: from='client.44297 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T08:22:59.940 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:22:59 smithi028 ceph-mon[64115]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-09-16T08:22:59.940 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:22:59 smithi028 ceph-mon[64115]: osdmap e118: 8 total, 7 up, 8 in 2024-09-16T08:22:59.940 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:22:59 smithi028 ceph-mon[64115]: from='osd.4 [v2:172.21.15.144:6800/1573842275,v1:172.21.15.144:6801/1573842275]' entity='osd.4' cmd={"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi144", "root=default"]} : dispatch 2024-09-16T08:22:59.940 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:22:59 smithi028 ceph-mon[64115]: from='osd.4 ' entity='osd.4' cmd={"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi144", "root=default"]} : dispatch 2024-09-16T08:22:59.941 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:22:59 smithi028 ceph-mon[64115]: pgmap v205: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 340 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 8.2 KiB/s rd, 4.9 MiB/s wr, 450 op/s; 241/1602 objects degraded (15.044%) 2024-09-16T08:22:59.941 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:22:59 smithi028 ceph-mon[64115]: from='osd.4 ' entity='osd.4' 2024-09-16T08:22:59.941 INFO:teuthology.orchestra.run.smithi028.stdout:{ 2024-09-16T08:22:59.941 INFO:teuthology.orchestra.run.smithi028.stdout: "mon": { 2024-09-16T08:22:59.941 INFO:teuthology.orchestra.run.smithi028.stdout: "ceph version 19.3.0-4917-g8293d73f (8293d73f8690540e843a81caec373f9cc29cf705) squid (dev)": 2 2024-09-16T08:22:59.941 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-09-16T08:22:59.941 INFO:teuthology.orchestra.run.smithi028.stdout: "mgr": { 2024-09-16T08:22:59.941 INFO:teuthology.orchestra.run.smithi028.stdout: "ceph version 19.3.0-4917-g8293d73f (8293d73f8690540e843a81caec373f9cc29cf705) squid (dev)": 2 2024-09-16T08:22:59.941 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-09-16T08:22:59.941 INFO:teuthology.orchestra.run.smithi028.stdout: "osd": { 2024-09-16T08:22:59.941 INFO:teuthology.orchestra.run.smithi028.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3, 2024-09-16T08:22:59.941 INFO:teuthology.orchestra.run.smithi028.stdout: "ceph version 19.3.0-4917-g8293d73f (8293d73f8690540e843a81caec373f9cc29cf705) squid (dev)": 4 2024-09-16T08:22:59.942 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-09-16T08:22:59.942 INFO:teuthology.orchestra.run.smithi028.stdout: "mds": { 2024-09-16T08:22:59.942 INFO:teuthology.orchestra.run.smithi028.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-16T08:22:59.942 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-09-16T08:22:59.942 INFO:teuthology.orchestra.run.smithi028.stdout: "overall": { 2024-09-16T08:22:59.942 INFO:teuthology.orchestra.run.smithi028.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 5, 2024-09-16T08:22:59.942 INFO:teuthology.orchestra.run.smithi028.stdout: "ceph version 19.3.0-4917-g8293d73f (8293d73f8690540e843a81caec373f9cc29cf705) squid (dev)": 8 2024-09-16T08:22:59.942 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-09-16T08:22:59.942 INFO:teuthology.orchestra.run.smithi028.stdout:} 2024-09-16T08:23:00.082 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:22:59 smithi144 ceph-mon[49149]: from='client.44297 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T08:23:00.082 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:22:59 smithi144 ceph-mon[49149]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-09-16T08:23:00.082 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:22:59 smithi144 ceph-mon[49149]: osdmap e118: 8 total, 7 up, 8 in 2024-09-16T08:23:00.082 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:22:59 smithi144 ceph-mon[49149]: from='osd.4 [v2:172.21.15.144:6800/1573842275,v1:172.21.15.144:6801/1573842275]' entity='osd.4' cmd={"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi144", "root=default"]} : dispatch 2024-09-16T08:23:00.082 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:22:59 smithi144 ceph-mon[49149]: from='osd.4 ' entity='osd.4' cmd={"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi144", "root=default"]} : dispatch 2024-09-16T08:23:00.082 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:22:59 smithi144 ceph-mon[49149]: pgmap v205: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 340 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 8.2 KiB/s rd, 4.9 MiB/s wr, 450 op/s; 241/1602 objects degraded (15.044%) 2024-09-16T08:23:00.082 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:22:59 smithi144 ceph-mon[49149]: from='osd.4 ' entity='osd.4' 2024-09-16T08:23:00.086 INFO:teuthology.orchestra.run.smithi028.stdout: 5 240342 9.67 MB/sec execute 478 sec latency 42.221 ms 2024-09-16T08:23:00.441 INFO:teuthology.orchestra.run.smithi028.stdout:{ 2024-09-16T08:23:00.441 INFO:teuthology.orchestra.run.smithi028.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:f7ad55f9ccd17ba66aadb19b4d291e67b60ea24e5d8a0c0e92941fd59c81e4ce", 2024-09-16T08:23:00.441 INFO:teuthology.orchestra.run.smithi028.stdout: "in_progress": true, 2024-09-16T08:23:00.441 INFO:teuthology.orchestra.run.smithi028.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-16T08:23:00.441 INFO:teuthology.orchestra.run.smithi028.stdout: "services_complete": [ 2024-09-16T08:23:00.441 INFO:teuthology.orchestra.run.smithi028.stdout: "crash", 2024-09-16T08:23:00.441 INFO:teuthology.orchestra.run.smithi028.stdout: "mon", 2024-09-16T08:23:00.441 INFO:teuthology.orchestra.run.smithi028.stdout: "mgr" 2024-09-16T08:23:00.442 INFO:teuthology.orchestra.run.smithi028.stdout: ], 2024-09-16T08:23:00.442 INFO:teuthology.orchestra.run.smithi028.stdout: "progress": "11/23 daemons upgraded", 2024-09-16T08:23:00.442 INFO:teuthology.orchestra.run.smithi028.stdout: "message": "Currently upgrading osd daemons", 2024-09-16T08:23:00.442 INFO:teuthology.orchestra.run.smithi028.stdout: "is_paused": false 2024-09-16T08:23:00.442 INFO:teuthology.orchestra.run.smithi028.stdout:} 2024-09-16T08:23:00.892 INFO:teuthology.orchestra.run.smithi028.stdout:HEALTH_WARN Degraded data redundancy: 241/1602 objects degraded (15.044%), 28 pgs degraded 2024-09-16T08:23:00.892 INFO:teuthology.orchestra.run.smithi028.stdout:[WRN] PG_DEGRADED: Degraded data redundancy: 241/1602 objects degraded (15.044%), 28 pgs degraded 2024-09-16T08:23:00.892 INFO:teuthology.orchestra.run.smithi028.stdout: pg 2.5 is active+undersized+degraded, acting [7,0] 2024-09-16T08:23:00.892 INFO:teuthology.orchestra.run.smithi028.stdout: pg 2.6 is active+undersized+degraded, acting [1,6] 2024-09-16T08:23:00.893 INFO:teuthology.orchestra.run.smithi028.stdout: pg 2.b is active+undersized+degraded, acting [7,5] 2024-09-16T08:23:00.893 INFO:teuthology.orchestra.run.smithi028.stdout: pg 2.d is active+undersized+degraded, acting [1,3] 2024-09-16T08:23:00.893 INFO:teuthology.orchestra.run.smithi028.stdout: pg 2.f is active+undersized+degraded, acting [0,7] 2024-09-16T08:23:00.893 INFO:teuthology.orchestra.run.smithi028.stdout: pg 2.11 is active+undersized+degraded, acting [6,1] 2024-09-16T08:23:00.893 INFO:teuthology.orchestra.run.smithi028.stdout: pg 2.13 is active+undersized+degraded, acting [0,3] 2024-09-16T08:23:00.893 INFO:teuthology.orchestra.run.smithi028.stdout: pg 2.18 is active+undersized+degraded, acting [5,7] 2024-09-16T08:23:00.893 INFO:teuthology.orchestra.run.smithi028.stdout: pg 2.1a is active+undersized+degraded, acting [6,7] 2024-09-16T08:23:00.893 INFO:teuthology.orchestra.run.smithi028.stdout: pg 2.1f is active+undersized+degraded, acting [0,7] 2024-09-16T08:23:00.893 INFO:teuthology.orchestra.run.smithi028.stdout: pg 3.1 is active+undersized+degraded, acting [0,3] 2024-09-16T08:23:00.893 INFO:teuthology.orchestra.run.smithi028.stdout: pg 3.3 is active+undersized+degraded, acting [0,6] 2024-09-16T08:23:00.893 INFO:teuthology.orchestra.run.smithi028.stdout: pg 3.6 is active+undersized+degraded, acting [0,1] 2024-09-16T08:23:00.893 INFO:teuthology.orchestra.run.smithi028.stdout: pg 3.9 is active+undersized+degraded, acting [2,7] 2024-09-16T08:23:00.893 INFO:teuthology.orchestra.run.smithi028.stdout: pg 3.a is active+undersized+degraded, acting [6,1] 2024-09-16T08:23:00.894 INFO:teuthology.orchestra.run.smithi028.stdout: pg 3.b is active+undersized+degraded, acting [3,0] 2024-09-16T08:23:00.894 INFO:teuthology.orchestra.run.smithi028.stdout: pg 3.e is active+undersized+degraded, acting [7,1] 2024-09-16T08:23:00.894 INFO:teuthology.orchestra.run.smithi028.stdout: pg 3.f is active+undersized+degraded, acting [7,0] 2024-09-16T08:23:00.894 INFO:teuthology.orchestra.run.smithi028.stdout: pg 3.11 is active+undersized+degraded, acting [7,6] 2024-09-16T08:23:00.894 INFO:teuthology.orchestra.run.smithi028.stdout: pg 3.13 is active+undersized+degraded, acting [7,2] 2024-09-16T08:23:00.894 INFO:teuthology.orchestra.run.smithi028.stdout: pg 3.14 is active+undersized+degraded, acting [7,6] 2024-09-16T08:23:00.894 INFO:teuthology.orchestra.run.smithi028.stdout: pg 3.15 is active+undersized+degraded, acting [7,3] 2024-09-16T08:23:00.894 INFO:teuthology.orchestra.run.smithi028.stdout: pg 3.19 is active+undersized+degraded, acting [1,3] 2024-09-16T08:23:00.894 INFO:teuthology.orchestra.run.smithi028.stdout: pg 3.1a is active+undersized+degraded, acting [1,2] 2024-09-16T08:23:00.894 INFO:teuthology.orchestra.run.smithi028.stdout: pg 3.1b is active+undersized+degraded, acting [0,7] 2024-09-16T08:23:00.894 INFO:teuthology.orchestra.run.smithi028.stdout: pg 3.1c is active+undersized+degraded, acting [5,1] 2024-09-16T08:23:00.894 INFO:teuthology.orchestra.run.smithi028.stdout: pg 3.1d is active+undersized+degraded, acting [5,6] 2024-09-16T08:23:00.894 INFO:teuthology.orchestra.run.smithi028.stdout: pg 4.5 is active+undersized+degraded, acting [6,7] 2024-09-16T08:23:01.082 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:23:00 smithi144 ceph-mon[49149]: from='client.34362 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T08:23:01.082 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:23:00 smithi144 ceph-mon[49149]: Health check update: Degraded data redundancy: 241/1602 objects degraded (15.044%), 28 pgs degraded (PG_DEGRADED) 2024-09-16T08:23:01.082 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:23:00 smithi144 ceph-mon[49149]: from='client.? 172.21.15.28:0/1144048908' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-16T08:23:01.082 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:23:00 smithi144 ceph-mon[49149]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-16T08:23:01.082 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:23:00 smithi144 ceph-mon[49149]: osd.4 [v2:172.21.15.144:6800/1573842275,v1:172.21.15.144:6801/1573842275] boot 2024-09-16T08:23:01.082 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:23:00 smithi144 ceph-mon[49149]: osdmap e119: 8 total, 8 up, 8 in 2024-09-16T08:23:01.082 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:23:00 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "osd metadata", "id": 4} : dispatch 2024-09-16T08:23:01.086 INFO:teuthology.orchestra.run.smithi028.stdout: 5 240369 9.65 MB/sec execute 479 sec latency 915.931 ms 2024-09-16T08:23:01.197 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:23:00 smithi028 ceph-mon[64115]: from='client.34362 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T08:23:01.197 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:23:00 smithi028 ceph-mon[64115]: Health check update: Degraded data redundancy: 241/1602 objects degraded (15.044%), 28 pgs degraded (PG_DEGRADED) 2024-09-16T08:23:01.197 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:23:00 smithi028 ceph-mon[64115]: from='client.? 172.21.15.28:0/1144048908' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-16T08:23:01.198 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:23:00 smithi028 ceph-mon[64115]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-16T08:23:01.198 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:23:00 smithi028 ceph-mon[64115]: osd.4 [v2:172.21.15.144:6800/1573842275,v1:172.21.15.144:6801/1573842275] boot 2024-09-16T08:23:01.198 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:23:00 smithi028 ceph-mon[64115]: osdmap e119: 8 total, 8 up, 8 in 2024-09-16T08:23:01.198 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:23:00 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "osd metadata", "id": 4} : dispatch 2024-09-16T08:23:02.082 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:23:01 smithi144 ceph-mon[49149]: from='client.44309 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T08:23:02.082 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:23:01 smithi144 ceph-mon[49149]: pgmap v207: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 343 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 9.1 KiB/s rd, 5.0 MiB/s wr, 464 op/s; 257/1656 objects degraded (15.519%) 2024-09-16T08:23:02.082 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:23:01 smithi144 ceph-mon[49149]: from='client.? 172.21.15.28:0/3599996452' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-16T08:23:02.082 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:23:01 smithi144 ceph-mon[49149]: osdmap e120: 8 total, 8 up, 8 in 2024-09-16T08:23:02.086 INFO:teuthology.orchestra.run.smithi028.stdout: 5 240369 9.63 MB/sec execute 480 sec latency 1915.984 ms 2024-09-16T08:23:02.197 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:23:01 smithi028 ceph-mon[64115]: from='client.44309 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T08:23:02.197 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:23:01 smithi028 ceph-mon[64115]: pgmap v207: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 343 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 9.1 KiB/s rd, 5.0 MiB/s wr, 464 op/s; 257/1656 objects degraded (15.519%) 2024-09-16T08:23:02.197 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:23:01 smithi028 ceph-mon[64115]: from='client.? 172.21.15.28:0/3599996452' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-16T08:23:02.197 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:23:01 smithi028 ceph-mon[64115]: osdmap e120: 8 total, 8 up, 8 in 2024-09-16T08:23:03.087 INFO:teuthology.orchestra.run.smithi028.stdout: 5 240547 9.63 MB/sec execute 481 sec latency 2089.861 ms 2024-09-16T08:23:03.310 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:23:03 smithi028 ceph-mon[64115]: osdmap e121: 8 total, 8 up, 8 in 2024-09-16T08:23:03.310 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:23:03 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:23:03.310 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:23:03 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-16T08:23:03.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:23:03 smithi144 ceph-mon[49149]: osdmap e121: 8 total, 8 up, 8 in 2024-09-16T08:23:03.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:23:03 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:23:03.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:23:03 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-16T08:23:04.087 INFO:teuthology.orchestra.run.smithi028.stdout: 5 240737 9.62 MB/sec execute 482 sec latency 669.935 ms 2024-09-16T08:23:04.364 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:23:04 smithi028 ceph-mon[64115]: pgmap v210: 97 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+remapped, 12 active+undersized, 19 active+undersized+degraded, 62 active+clean; 342 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 12 KiB/s rd, 4.5 MiB/s wr, 501 op/s; 235/1671 objects degraded (14.063%); 939 KiB/s, 62 keys/s, 10 objects/s recovering 2024-09-16T08:23:04.364 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:23:04 smithi028 ceph-mon[64115]: osdmap e122: 8 total, 8 up, 8 in 2024-09-16T08:23:04.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:23:04 smithi144 ceph-mon[49149]: pgmap v210: 97 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+remapped, 12 active+undersized, 19 active+undersized+degraded, 62 active+clean; 342 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 12 KiB/s rd, 4.5 MiB/s wr, 501 op/s; 235/1671 objects degraded (14.063%); 939 KiB/s, 62 keys/s, 10 objects/s recovering 2024-09-16T08:23:04.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:23:04 smithi144 ceph-mon[49149]: osdmap e122: 8 total, 8 up, 8 in 2024-09-16T08:23:05.087 INFO:teuthology.orchestra.run.smithi028.stdout: 5 240892 9.60 MB/sec execute 483 sec latency 924.210 ms 2024-09-16T08:23:05.380 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:23:05 smithi028 ceph-mon[64115]: osdmap e123: 8 total, 8 up, 8 in 2024-09-16T08:23:05.380 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:23:05 smithi028 ceph-mon[64115]: pgmap v213: 97 pgs: 3 peering, 6 active+recovery_wait+undersized+degraded+remapped, 3 active+undersized+remapped, 1 active+undersized, 5 active+undersized+degraded, 79 active+clean; 340 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 5.5 KiB/s rd, 2.9 MiB/s wr, 302 op/s; 201/1650 objects degraded (12.182%); 1.9 MiB/s, 65 keys/s, 30 objects/s recovering 2024-09-16T08:23:05.380 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:23:05 smithi028 ceph-mon[64115]: Health check update: Degraded data redundancy: 235/1671 objects degraded (14.063%), 21 pgs degraded (PG_DEGRADED) 2024-09-16T08:23:05.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:23:05 smithi144 ceph-mon[49149]: osdmap e123: 8 total, 8 up, 8 in 2024-09-16T08:23:05.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:23:05 smithi144 ceph-mon[49149]: pgmap v213: 97 pgs: 3 peering, 6 active+recovery_wait+undersized+degraded+remapped, 3 active+undersized+remapped, 1 active+undersized, 5 active+undersized+degraded, 79 active+clean; 340 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 5.5 KiB/s rd, 2.9 MiB/s wr, 302 op/s; 201/1650 objects degraded (12.182%); 1.9 MiB/s, 65 keys/s, 30 objects/s recovering 2024-09-16T08:23:05.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:23:05 smithi144 ceph-mon[49149]: Health check update: Degraded data redundancy: 235/1671 objects degraded (14.063%), 21 pgs degraded (PG_DEGRADED) 2024-09-16T08:23:06.087 INFO:teuthology.orchestra.run.smithi028.stdout: 5 241075 9.58 MB/sec execute 484 sec latency 992.970 ms 2024-09-16T08:23:06.387 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:23:06 smithi028 ceph-mon[64115]: osdmap e124: 8 total, 8 up, 8 in 2024-09-16T08:23:06.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:23:06 smithi144 ceph-mon[49149]: osdmap e124: 8 total, 8 up, 8 in 2024-09-16T08:23:07.087 INFO:teuthology.orchestra.run.smithi028.stdout: 5 241263 9.58 MB/sec execute 485 sec latency 50.021 ms 2024-09-16T08:23:07.400 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:23:07 smithi028 ceph-mon[64115]: osdmap e125: 8 total, 8 up, 8 in 2024-09-16T08:23:07.400 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:23:07 smithi028 ceph-mon[64115]: pgmap v216: 97 pgs: 5 peering, 6 active+recovery_wait+undersized+degraded+remapped, 3 active+undersized+remapped, 83 active+clean; 341 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 1.5 MiB/s wr, 64 op/s; 148/1647 objects degraded (8.986%); 1.0 MiB/s, 39 objects/s recovering 2024-09-16T08:23:07.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:23:07 smithi144 ceph-mon[49149]: osdmap e125: 8 total, 8 up, 8 in 2024-09-16T08:23:07.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:23:07 smithi144 ceph-mon[49149]: pgmap v216: 97 pgs: 5 peering, 6 active+recovery_wait+undersized+degraded+remapped, 3 active+undersized+remapped, 83 active+clean; 341 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 1.5 MiB/s wr, 64 op/s; 148/1647 objects degraded (8.986%); 1.0 MiB/s, 39 objects/s recovering 2024-09-16T08:23:08.087 INFO:teuthology.orchestra.run.smithi028.stdout: 5 241530 9.57 MB/sec execute 486 sec latency 615.785 ms 2024-09-16T08:23:08.435 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:23:08 smithi028 ceph-mon[64115]: osdmap e126: 8 total, 8 up, 8 in 2024-09-16T08:23:08.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:23:08 smithi144 ceph-mon[49149]: osdmap e126: 8 total, 8 up, 8 in 2024-09-16T08:23:09.087 INFO:teuthology.orchestra.run.smithi028.stdout: 5 241881 9.57 MB/sec execute 487 sec latency 853.615 ms 2024-09-16T08:23:09.451 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:23:09 smithi028 ceph-mon[64115]: osdmap e127: 8 total, 8 up, 8 in 2024-09-16T08:23:09.451 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:23:09 smithi028 ceph-mon[64115]: pgmap v219: 97 pgs: 3 peering, 94 active+clean; 339 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 7.7 KiB/s rd, 3.2 MiB/s wr, 359 op/s; 472 KiB/s, 73 objects/s recovering 2024-09-16T08:23:09.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:23:09 smithi144 ceph-mon[49149]: osdmap e127: 8 total, 8 up, 8 in 2024-09-16T08:23:09.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:23:09 smithi144 ceph-mon[49149]: pgmap v219: 97 pgs: 3 peering, 94 active+clean; 339 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 7.7 KiB/s rd, 3.2 MiB/s wr, 359 op/s; 472 KiB/s, 73 objects/s recovering 2024-09-16T08:23:10.087 INFO:teuthology.orchestra.run.smithi028.stdout: 5 242315 9.56 MB/sec execute 488 sec latency 54.598 ms 2024-09-16T08:23:10.466 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:23:10 smithi028 ceph-mon[64115]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 148/1647 objects degraded (8.986%), 6 pgs degraded) 2024-09-16T08:23:10.466 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:23:10 smithi028 ceph-mon[64115]: Cluster is now healthy 2024-09-16T08:23:10.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:23:10 smithi144 ceph-mon[49149]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 148/1647 objects degraded (8.986%), 6 pgs degraded) 2024-09-16T08:23:10.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:23:10 smithi144 ceph-mon[49149]: Cluster is now healthy 2024-09-16T08:23:11.087 INFO:teuthology.orchestra.run.smithi028.stdout: 5 242695 9.56 MB/sec execute 489 sec latency 40.095 ms 2024-09-16T08:23:11.476 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:23:11 smithi028 ceph-mon[64115]: pgmap v220: 97 pgs: 1 peering, 96 active+clean; 336 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 5.9 KiB/s rd, 3.0 MiB/s wr, 309 op/s; 260 KiB/s, 38 objects/s recovering 2024-09-16T08:23:11.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:23:11 smithi144 ceph-mon[49149]: pgmap v220: 97 pgs: 1 peering, 96 active+clean; 336 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 5.9 KiB/s rd, 3.0 MiB/s wr, 309 op/s; 260 KiB/s, 38 objects/s recovering 2024-09-16T08:23:12.087 INFO:teuthology.orchestra.run.smithi028.stdout: 5 243313 9.59 MB/sec execute 490 sec latency 44.344 ms 2024-09-16T08:23:12.484 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:23:12 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16} : dispatch 2024-09-16T08:23:12.809 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:23:12 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16} : dispatch 2024-09-16T08:23:13.087 INFO:teuthology.orchestra.run.smithi028.stdout: 5 243901 9.61 MB/sec execute 491 sec latency 32.706 ms 2024-09-16T08:23:13.493 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:23:13 smithi028 ceph-mon[64115]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-09-16T08:23:13.493 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:23:13 smithi028 ceph-mon[64115]: Upgrade: osd.5 is safe to restart 2024-09-16T08:23:13.493 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:23:13 smithi028 ceph-mon[64115]: pgmap v221: 97 pgs: 1 peering, 96 active+clean; 338 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 8.9 KiB/s rd, 4.3 MiB/s wr, 609 op/s; 221 KiB/s, 32 objects/s recovering 2024-09-16T08:23:13.832 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:23:13 smithi144 ceph-mon[49149]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-09-16T08:23:13.832 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:23:13 smithi144 ceph-mon[49149]: Upgrade: osd.5 is safe to restart 2024-09-16T08:23:13.832 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:23:13 smithi144 ceph-mon[49149]: pgmap v221: 97 pgs: 1 peering, 96 active+clean; 338 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 8.9 KiB/s rd, 4.3 MiB/s wr, 609 op/s; 221 KiB/s, 32 objects/s recovering 2024-09-16T08:23:14.087 INFO:teuthology.orchestra.run.smithi028.stdout: 5 244208 9.61 MB/sec execute 492 sec latency 30.430 ms 2024-09-16T08:23:15.087 INFO:teuthology.orchestra.run.smithi028.stdout: 5 244566 9.60 MB/sec execute 493 sec latency 39.341 ms 2024-09-16T08:23:15.267 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:23:15 smithi028 ceph-mon[64115]: Upgrade: Updating osd.5 2024-09-16T08:23:15.267 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:23:15 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:23:15.267 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:23:15 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "auth get", "entity": "osd.5"} : dispatch 2024-09-16T08:23:15.267 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:23:15 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-16T08:23:15.267 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:23:15 smithi028 ceph-mon[64115]: Deploying daemon osd.5 on smithi144 2024-09-16T08:23:15.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:23:15 smithi144 ceph-mon[49149]: Upgrade: Updating osd.5 2024-09-16T08:23:15.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:23:15 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:23:15.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:23:15 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "auth get", "entity": "osd.5"} : dispatch 2024-09-16T08:23:15.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:23:15 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-16T08:23:15.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:23:15 smithi144 ceph-mon[49149]: Deploying daemon osd.5 on smithi144 2024-09-16T08:23:16.087 INFO:teuthology.orchestra.run.smithi028.stdout: 5 244878 9.60 MB/sec execute 494 sec latency 39.586 ms 2024-09-16T08:23:16.305 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:23:16 smithi028 ceph-mon[64115]: pgmap v222: 97 pgs: 97 active+clean; 339 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 7.4 KiB/s rd, 5.8 MiB/s wr, 628 op/s; 182 KiB/s, 26 objects/s recovering 2024-09-16T08:23:16.409 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:23:16 smithi144 ceph-mon[49149]: pgmap v222: 97 pgs: 97 active+clean; 339 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 7.4 KiB/s rd, 5.8 MiB/s wr, 628 op/s; 182 KiB/s, 26 objects/s recovering 2024-09-16T08:23:17.087 INFO:teuthology.orchestra.run.smithi028.stdout: 5 245164 9.60 MB/sec execute 495 sec latency 48.942 ms 2024-09-16T08:23:17.946 INFO:journalctl@ceph.osd.5.smithi144.stdout:Sep 16 08:23:17 smithi144 systemd[1]: Stopping Ceph osd.5 for 5622b2fe-7402-11ef-bceb-c7b262605968... 2024-09-16T08:23:18.087 INFO:teuthology.orchestra.run.smithi028.stdout: 5 245472 9.59 MB/sec execute 496 sec latency 64.385 ms 2024-09-16T08:23:18.228 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:23:18 smithi028 ceph-mon[64115]: pgmap v223: 97 pgs: 97 active+clean; 342 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 5.4 MiB/s wr, 547 op/s; 152 KiB/s, 22 objects/s recovering 2024-09-16T08:23:18.228 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:23:18 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:23:18.228 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:23:18 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-16T08:23:18.228 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:23:18 smithi028 ceph-mon[64115]: osd.5 marked itself down and dead 2024-09-16T08:23:18.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:23:18 smithi144 ceph-mon[49149]: pgmap v223: 97 pgs: 97 active+clean; 342 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 5.4 MiB/s wr, 547 op/s; 152 KiB/s, 22 objects/s recovering 2024-09-16T08:23:18.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:23:18 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:23:18.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:23:18 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-16T08:23:18.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:23:18 smithi144 ceph-mon[49149]: osd.5 marked itself down and dead 2024-09-16T08:23:18.332 INFO:journalctl@ceph.osd.5.smithi144.stdout:Sep 16 08:23:17 smithi144 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-5[32740]: 2024-09-16T08:23:17.945+0000 7f3f4715d700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.5 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-16T08:23:18.332 INFO:journalctl@ceph.osd.5.smithi144.stdout:Sep 16 08:23:17 smithi144 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-5[32740]: 2024-09-16T08:23:17.945+0000 7f3f4715d700 -1 osd.5 127 *** Got signal Terminated *** 2024-09-16T08:23:18.332 INFO:journalctl@ceph.osd.5.smithi144.stdout:Sep 16 08:23:17 smithi144 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-5[32740]: 2024-09-16T08:23:17.945+0000 7f3f4715d700 -1 osd.5 127 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-16T08:23:19.082 INFO:journalctl@ceph.osd.5.smithi144.stdout:Sep 16 08:23:18 smithi144 podman[54621]: 2024-09-16 08:23:18.79980946 +0000 UTC m=+0.986294724 container died 70452c145c6015956bed052b9a7051f3ca99e6268e31550cc7d46f267e7a4945 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-5, vcs-type=git, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.openshift.expose-services=, version=8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_BRANCH=HEAD, com.redhat.component=centos-stream-container, io.buildah.version=1.19.8, vendor=Red Hat, Inc., GIT_REPO=https://github.com/ceph/ceph-container.git, release=754, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, architecture=x86_64, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, ceph=True, RELEASE=HEAD, io.openshift.tags=base centos centos-stream, distribution-scope=public, io.k8s.display-name=CentOS Stream 8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, maintainer=Guillaume Abrioux , GIT_CLEAN=True, build-date=2022-05-03T08:36:31.336870, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, CEPH_POINT_RELEASE=-17.2.0, name=centos-stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly.) 2024-09-16T08:23:19.087 INFO:teuthology.orchestra.run.smithi028.stdout: 5 245632 9.58 MB/sec execute 497 sec latency 781.203 ms 2024-09-16T08:23:19.403 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:23:19 smithi028 ceph-mon[64115]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-16T08:23:19.403 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:23:19 smithi028 ceph-mon[64115]: osdmap e128: 8 total, 7 up, 8 in 2024-09-16T08:23:19.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:23:19 smithi144 ceph-mon[49149]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-16T08:23:19.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:23:19 smithi144 ceph-mon[49149]: osdmap e128: 8 total, 7 up, 8 in 2024-09-16T08:23:20.082 INFO:journalctl@ceph.osd.5.smithi144.stdout:Sep 16 08:23:19 smithi144 podman[54621]: 2024-09-16 08:23:19.603924056 +0000 UTC m=+1.790409320 container cleanup 70452c145c6015956bed052b9a7051f3ca99e6268e31550cc7d46f267e7a4945 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-5, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, release=754, vcs-type=git, vendor=Red Hat, Inc., com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., build-date=2022-05-03T08:36:31.336870, io.buildah.version=1.19.8, maintainer=Guillaume Abrioux , ceph=True, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_BRANCH=HEAD, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, architecture=x86_64, io.openshift.expose-services=, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_REPO=https://github.com/ceph/ceph-container.git, io.k8s.display-name=CentOS Stream 8, RELEASE=HEAD, com.redhat.component=centos-stream-container, io.openshift.tags=base centos centos-stream, version=8, name=centos-stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_CLEAN=True, distribution-scope=public, CEPH_POINT_RELEASE=-17.2.0) 2024-09-16T08:23:20.082 INFO:journalctl@ceph.osd.5.smithi144.stdout:Sep 16 08:23:19 smithi144 bash[54621]: ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-5 2024-09-16T08:23:20.082 INFO:journalctl@ceph.osd.5.smithi144.stdout:Sep 16 08:23:19 smithi144 podman[54636]: 2024-09-16 08:23:19.847103184 +0000 UTC m=+1.050651132 container remove 70452c145c6015956bed052b9a7051f3ca99e6268e31550cc7d46f267e7a4945 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-5, GIT_REPO=https://github.com/ceph/ceph-container.git, distribution-scope=public, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., name=centos-stream, GIT_CLEAN=True, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_BRANCH=HEAD, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vcs-type=git, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.k8s.display-name=CentOS Stream 8, com.redhat.component=centos-stream-container, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, RELEASE=HEAD, ceph=True, build-date=2022-05-03T08:36:31.336870, version=8, maintainer=Guillaume Abrioux , vendor=Red Hat, Inc., io.openshift.tags=base centos centos-stream, io.openshift.expose-services=, architecture=x86_64, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.buildah.version=1.19.8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, CEPH_POINT_RELEASE=-17.2.0, release=754) 2024-09-16T08:23:20.087 INFO:teuthology.orchestra.run.smithi028.stdout: 5 245888 9.56 MB/sec execute 498 sec latency 1052.226 ms 2024-09-16T08:23:20.175 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:23:20 smithi028 ceph-mon[64115]: pgmap v225: 97 pgs: 4 peering, 10 stale+active+clean, 83 active+clean; 339 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 8.2 KiB/s rd, 5.0 MiB/s wr, 453 op/s 2024-09-16T08:23:20.175 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:23:20 smithi028 ceph-mon[64115]: osdmap e129: 8 total, 7 up, 8 in 2024-09-16T08:23:20.510 INFO:journalctl@ceph.osd.5.smithi144.stdout:Sep 16 08:23:20 smithi144 podman[54724]: 2024-09-16 08:23:20.106853932 +0000 UTC m=+0.016257758 image pull 425e316ba8ebf74f04aa3640f7deb4fa3f293a78d0c8d097cb679eaf86ddca6c quay.ceph.io/ceph-ci/ceph@sha256:f7ad55f9ccd17ba66aadb19b4d291e67b60ea24e5d8a0c0e92941fd59c81e4ce 2024-09-16T08:23:20.510 INFO:journalctl@ceph.osd.5.smithi144.stdout:Sep 16 08:23:20 smithi144 podman[54724]: 2024-09-16 08:23:20.240058066 +0000 UTC m=+0.149461894 container create df989250074bfc0e69eb40217c612543c3e4a6054fb2f6c7f1f193dc16efa0d5 (image=quay.ceph.io/ceph-ci/ceph@sha256:f7ad55f9ccd17ba66aadb19b4d291e67b60ea24e5d8a0c0e92941fd59c81e4ce, name=ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-5-deactivate, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, io.buildah.version=1.37.2, ceph=True, RELEASE=wip-guits-main-2024-09-13-1248-8293d73, org.label-schema.build-date=20240731, GIT_CLEAN=True, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux ) 2024-09-16T08:23:20.510 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:23:20 smithi144 ceph-mon[49149]: pgmap v225: 97 pgs: 4 peering, 10 stale+active+clean, 83 active+clean; 339 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 8.2 KiB/s rd, 5.0 MiB/s wr, 453 op/s 2024-09-16T08:23:20.510 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:23:20 smithi144 ceph-mon[49149]: osdmap e129: 8 total, 7 up, 8 in 2024-09-16T08:23:20.832 INFO:journalctl@ceph.osd.5.smithi144.stdout:Sep 16 08:23:20 smithi144 podman[54724]: 2024-09-16 08:23:20.509725063 +0000 UTC m=+0.419128890 container init df989250074bfc0e69eb40217c612543c3e4a6054fb2f6c7f1f193dc16efa0d5 (image=quay.ceph.io/ceph-ci/ceph@sha256:f7ad55f9ccd17ba66aadb19b4d291e67b60ea24e5d8a0c0e92941fd59c81e4ce, name=ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-5-deactivate, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-guits-main-2024-09-13-1248-8293d73, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965) 2024-09-16T08:23:20.832 INFO:journalctl@ceph.osd.5.smithi144.stdout:Sep 16 08:23:20 smithi144 podman[54724]: 2024-09-16 08:23:20.51407942 +0000 UTC m=+0.423483254 container start df989250074bfc0e69eb40217c612543c3e4a6054fb2f6c7f1f193dc16efa0d5 (image=quay.ceph.io/ceph-ci/ceph@sha256:f7ad55f9ccd17ba66aadb19b4d291e67b60ea24e5d8a0c0e92941fd59c81e4ce, name=ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-5-deactivate, io.buildah.version=1.37.2, ceph=True, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, CEPH_POINT_RELEASE=, RELEASE=wip-guits-main-2024-09-13-1248-8293d73, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.build-date=20240731) 2024-09-16T08:23:20.832 INFO:journalctl@ceph.osd.5.smithi144.stdout:Sep 16 08:23:20 smithi144 podman[54724]: 2024-09-16 08:23:20.601508383 +0000 UTC m=+0.510912215 container attach df989250074bfc0e69eb40217c612543c3e4a6054fb2f6c7f1f193dc16efa0d5 (image=quay.ceph.io/ceph-ci/ceph@sha256:f7ad55f9ccd17ba66aadb19b4d291e67b60ea24e5d8a0c0e92941fd59c81e4ce, name=ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-5-deactivate, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, RELEASE=wip-guits-main-2024-09-13-1248-8293d73, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, GIT_BRANCH=HEAD) 2024-09-16T08:23:20.832 INFO:journalctl@ceph.osd.5.smithi144.stdout:Sep 16 08:23:20 smithi144 podman[54724]: 2024-09-16 08:23:20.746177514 +0000 UTC m=+0.655581340 container died df989250074bfc0e69eb40217c612543c3e4a6054fb2f6c7f1f193dc16efa0d5 (image=quay.ceph.io/ceph-ci/ceph@sha256:f7ad55f9ccd17ba66aadb19b4d291e67b60ea24e5d8a0c0e92941fd59c81e4ce, name=ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-5-deactivate, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, RELEASE=wip-guits-main-2024-09-13-1248-8293d73, ceph=True, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, io.buildah.version=1.37.2) 2024-09-16T08:23:21.087 INFO:teuthology.orchestra.run.smithi028.stdout: 5 246127 9.55 MB/sec execute 499 sec latency 42.504 ms 2024-09-16T08:23:21.461 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:23:21 smithi028 ceph-mon[64115]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2024-09-16T08:23:21.461 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:23:21 smithi028 ceph-mon[64115]: Health check failed: Degraded data redundancy: 68/1611 objects degraded (4.221%), 6 pgs degraded (PG_DEGRADED) 2024-09-16T08:23:21.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:23:21 smithi144 ceph-mon[49149]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2024-09-16T08:23:21.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:23:21 smithi144 ceph-mon[49149]: Health check failed: Degraded data redundancy: 68/1611 objects degraded (4.221%), 6 pgs degraded (PG_DEGRADED) 2024-09-16T08:23:21.907 INFO:journalctl@ceph.osd.5.smithi144.stdout:Sep 16 08:23:21 smithi144 podman[54746]: 2024-09-16 08:23:21.640218126 +0000 UTC m=+0.887200577 container remove df989250074bfc0e69eb40217c612543c3e4a6054fb2f6c7f1f193dc16efa0d5 (image=quay.ceph.io/ceph-ci/ceph@sha256:f7ad55f9ccd17ba66aadb19b4d291e67b60ea24e5d8a0c0e92941fd59c81e4ce, name=ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-5-deactivate, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-guits-main-2024-09-13-1248-8293d73, ceph=True, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=) 2024-09-16T08:23:21.907 INFO:journalctl@ceph.osd.5.smithi144.stdout:Sep 16 08:23:21 smithi144 systemd[1]: ceph-5622b2fe-7402-11ef-bceb-c7b262605968@osd.5.service: Deactivated successfully. 2024-09-16T08:23:21.907 INFO:journalctl@ceph.osd.5.smithi144.stdout:Sep 16 08:23:21 smithi144 systemd[1]: Stopped Ceph osd.5 for 5622b2fe-7402-11ef-bceb-c7b262605968. 2024-09-16T08:23:21.908 INFO:journalctl@ceph.osd.5.smithi144.stdout:Sep 16 08:23:21 smithi144 systemd[1]: ceph-5622b2fe-7402-11ef-bceb-c7b262605968@osd.5.service: Consumed 4min 57.190s CPU time. 2024-09-16T08:23:22.087 INFO:teuthology.orchestra.run.smithi028.stdout: 5 246334 9.55 MB/sec execute 500 sec latency 74.029 ms 2024-09-16T08:23:22.177 INFO:journalctl@ceph.osd.5.smithi144.stdout:Sep 16 08:23:21 smithi144 systemd[1]: Starting Ceph osd.5 for 5622b2fe-7402-11ef-bceb-c7b262605968... 2024-09-16T08:23:22.177 INFO:journalctl@ceph.osd.5.smithi144.stdout:Sep 16 08:23:22 smithi144 podman[54840]: 2024-09-16 08:23:22.046805236 +0000 UTC m=+0.022537497 image pull 425e316ba8ebf74f04aa3640f7deb4fa3f293a78d0c8d097cb679eaf86ddca6c quay.ceph.io/ceph-ci/ceph@sha256:f7ad55f9ccd17ba66aadb19b4d291e67b60ea24e5d8a0c0e92941fd59c81e4ce 2024-09-16T08:23:22.255 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:23:22 smithi028 ceph-mon[64115]: pgmap v227: 97 pgs: 2 active+undersized, 7 peering, 5 stale+active+clean, 6 active+undersized+degraded, 77 active+clean; 337 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 7.0 KiB/s rd, 5.6 MiB/s wr, 458 op/s; 68/1611 objects degraded (4.221%) 2024-09-16T08:23:22.519 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:23:22 smithi144 ceph-mon[49149]: pgmap v227: 97 pgs: 2 active+undersized, 7 peering, 5 stale+active+clean, 6 active+undersized+degraded, 77 active+clean; 337 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 7.0 KiB/s rd, 5.6 MiB/s wr, 458 op/s; 68/1611 objects degraded (4.221%) 2024-09-16T08:23:22.519 INFO:journalctl@ceph.osd.5.smithi144.stdout:Sep 16 08:23:22 smithi144 podman[54840]: 2024-09-16 08:23:22.174631341 +0000 UTC m=+0.150363602 container create 916528738c4a3b3179c5f84f202d801bffb479fa6c2291dd22377658b411721a (image=quay.ceph.io/ceph-ci/ceph@sha256:f7ad55f9ccd17ba66aadb19b4d291e67b60ea24e5d8a0c0e92941fd59c81e4ce, name=ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-5-activate, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, RELEASE=wip-guits-main-2024-09-13-1248-8293d73, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, ceph=True, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965) 2024-09-16T08:23:22.520 INFO:journalctl@ceph.osd.5.smithi144.stdout:Sep 16 08:23:22 smithi144 podman[54840]: 2024-09-16 08:23:22.419810822 +0000 UTC m=+0.395543084 container init 916528738c4a3b3179c5f84f202d801bffb479fa6c2291dd22377658b411721a (image=quay.ceph.io/ceph-ci/ceph@sha256:f7ad55f9ccd17ba66aadb19b4d291e67b60ea24e5d8a0c0e92941fd59c81e4ce, name=ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-5-activate, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_CLEAN=True, io.buildah.version=1.37.2, RELEASE=wip-guits-main-2024-09-13-1248-8293d73, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2) 2024-09-16T08:23:22.520 INFO:journalctl@ceph.osd.5.smithi144.stdout:Sep 16 08:23:22 smithi144 podman[54840]: 2024-09-16 08:23:22.424582104 +0000 UTC m=+0.400314357 container start 916528738c4a3b3179c5f84f202d801bffb479fa6c2291dd22377658b411721a (image=quay.ceph.io/ceph-ci/ceph@sha256:f7ad55f9ccd17ba66aadb19b4d291e67b60ea24e5d8a0c0e92941fd59c81e4ce, name=ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-5-activate, GIT_CLEAN=True, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, RELEASE=wip-guits-main-2024-09-13-1248-8293d73, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , ceph=True) 2024-09-16T08:23:22.520 INFO:journalctl@ceph.osd.5.smithi144.stdout:Sep 16 08:23:22 smithi144 podman[54840]: 2024-09-16 08:23:22.519823899 +0000 UTC m=+0.495556161 container attach 916528738c4a3b3179c5f84f202d801bffb479fa6c2291dd22377658b411721a (image=quay.ceph.io/ceph-ci/ceph@sha256:f7ad55f9ccd17ba66aadb19b4d291e67b60ea24e5d8a0c0e92941fd59c81e4ce, name=ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-5-activate, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, RELEASE=wip-guits-main-2024-09-13-1248-8293d73, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=) 2024-09-16T08:23:22.832 INFO:journalctl@ceph.osd.5.smithi144.stdout:Sep 16 08:23:22 smithi144 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-5-activate[54854]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-16T08:23:22.832 INFO:journalctl@ceph.osd.5.smithi144.stdout:Sep 16 08:23:22 smithi144 bash[54840]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-16T08:23:22.832 INFO:journalctl@ceph.osd.5.smithi144.stdout:Sep 16 08:23:22 smithi144 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-5-activate[54854]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-16T08:23:22.832 INFO:journalctl@ceph.osd.5.smithi144.stdout:Sep 16 08:23:22 smithi144 bash[54840]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-16T08:23:23.087 INFO:teuthology.orchestra.run.smithi028.stdout: 5 246624 9.56 MB/sec execute 501 sec latency 47.163 ms 2024-09-16T08:23:23.546 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:23:23 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:23:23.546 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:23:23 smithi028 ceph-mon[64115]: pgmap v228: 97 pgs: 5 active+undersized, 7 peering, 3 stale+active+clean, 13 active+undersized+degraded, 69 active+clean; 339 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 8.6 KiB/s rd, 3.9 MiB/s wr, 430 op/s; 108/1602 objects degraded (6.742%) 2024-09-16T08:23:23.832 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:23:23 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:23:23.832 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:23:23 smithi144 ceph-mon[49149]: pgmap v228: 97 pgs: 5 active+undersized, 7 peering, 3 stale+active+clean, 13 active+undersized+degraded, 69 active+clean; 339 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 8.6 KiB/s rd, 3.9 MiB/s wr, 430 op/s; 108/1602 objects degraded (6.742%) 2024-09-16T08:23:24.087 INFO:teuthology.orchestra.run.smithi028.stdout: 5 246992 9.56 MB/sec execute 502 sec latency 78.507 ms 2024-09-16T08:23:25.087 INFO:teuthology.orchestra.run.smithi028.stdout: 5 247354 9.57 MB/sec execute 503 sec latency 63.649 ms 2024-09-16T08:23:25.139 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:23:24 smithi028 ceph-mon[64115]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2024-09-16T08:23:25.241 INFO:journalctl@ceph.osd.5.smithi144.stdout:Sep 16 08:23:24 smithi144 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-5-activate[54854]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2024-09-16T08:23:25.241 INFO:journalctl@ceph.osd.5.smithi144.stdout:Sep 16 08:23:24 smithi144 bash[54840]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2024-09-16T08:23:25.241 INFO:journalctl@ceph.osd.5.smithi144.stdout:Sep 16 08:23:24 smithi144 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-5-activate[54854]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-5 --no-mon-config --dev /dev/mapper/vg_nvme-lv_3 2024-09-16T08:23:25.241 INFO:journalctl@ceph.osd.5.smithi144.stdout:Sep 16 08:23:24 smithi144 bash[54840]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-5 --no-mon-config --dev /dev/mapper/vg_nvme-lv_3 2024-09-16T08:23:25.241 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:23:25 smithi144 ceph-mon[49149]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2024-09-16T08:23:25.582 INFO:journalctl@ceph.osd.5.smithi144.stdout:Sep 16 08:23:25 smithi144 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-5-activate[54854]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_3 2024-09-16T08:23:25.582 INFO:journalctl@ceph.osd.5.smithi144.stdout:Sep 16 08:23:25 smithi144 bash[54840]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_3 2024-09-16T08:23:25.583 INFO:journalctl@ceph.osd.5.smithi144.stdout:Sep 16 08:23:25 smithi144 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-5-activate[54854]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-09-16T08:23:25.583 INFO:journalctl@ceph.osd.5.smithi144.stdout:Sep 16 08:23:25 smithi144 bash[54840]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-09-16T08:23:25.583 INFO:journalctl@ceph.osd.5.smithi144.stdout:Sep 16 08:23:25 smithi144 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-5-activate[54854]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-5/block 2024-09-16T08:23:25.583 INFO:journalctl@ceph.osd.5.smithi144.stdout:Sep 16 08:23:25 smithi144 bash[54840]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-5/block 2024-09-16T08:23:25.583 INFO:journalctl@ceph.osd.5.smithi144.stdout:Sep 16 08:23:25 smithi144 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-5-activate[54854]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2024-09-16T08:23:25.583 INFO:journalctl@ceph.osd.5.smithi144.stdout:Sep 16 08:23:25 smithi144 bash[54840]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2024-09-16T08:23:25.583 INFO:journalctl@ceph.osd.5.smithi144.stdout:Sep 16 08:23:25 smithi144 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-5-activate[54854]: --> ceph-volume raw activate successful for osd ID: 5 2024-09-16T08:23:25.584 INFO:journalctl@ceph.osd.5.smithi144.stdout:Sep 16 08:23:25 smithi144 bash[54840]: --> ceph-volume raw activate successful for osd ID: 5 2024-09-16T08:23:25.584 INFO:journalctl@ceph.osd.5.smithi144.stdout:Sep 16 08:23:25 smithi144 podman[54840]: 2024-09-16 08:23:25.280077328 +0000 UTC m=+3.255809582 container died 916528738c4a3b3179c5f84f202d801bffb479fa6c2291dd22377658b411721a (image=quay.ceph.io/ceph-ci/ceph@sha256:f7ad55f9ccd17ba66aadb19b4d291e67b60ea24e5d8a0c0e92941fd59c81e4ce, name=ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-5-activate, RELEASE=wip-guits-main-2024-09-13-1248-8293d73, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.license=GPLv2) 2024-09-16T08:23:26.087 INFO:teuthology.orchestra.run.smithi028.stdout: 5 247588 9.56 MB/sec execute 504 sec latency 50.936 ms 2024-09-16T08:23:26.175 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:23:25 smithi028 ceph-mon[64115]: pgmap v229: 97 pgs: 9 active+undersized, 22 active+undersized+degraded, 66 active+clean; 333 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 8.6 KiB/s rd, 4.3 MiB/s wr, 484 op/s; 165/1557 objects degraded (10.597%) 2024-09-16T08:23:26.292 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:23:26 smithi144 ceph-mon[49149]: pgmap v229: 97 pgs: 9 active+undersized, 22 active+undersized+degraded, 66 active+clean; 333 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 8.6 KiB/s rd, 4.3 MiB/s wr, 484 op/s; 165/1557 objects degraded (10.597%) 2024-09-16T08:23:26.292 INFO:journalctl@ceph.osd.5.smithi144.stdout:Sep 16 08:23:25 smithi144 podman[54996]: 2024-09-16 08:23:25.932993454 +0000 UTC m=+0.645893796 container remove 916528738c4a3b3179c5f84f202d801bffb479fa6c2291dd22377658b411721a (image=quay.ceph.io/ceph-ci/ceph@sha256:f7ad55f9ccd17ba66aadb19b4d291e67b60ea24e5d8a0c0e92941fd59c81e4ce, name=ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-5-activate, RELEASE=wip-guits-main-2024-09-13-1248-8293d73, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, maintainer=Guillaume Abrioux , GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.license=GPLv2) 2024-09-16T08:23:26.292 INFO:journalctl@ceph.osd.5.smithi144.stdout:Sep 16 08:23:26 smithi144 podman[55052]: 2024-09-16 08:23:26.061115455 +0000 UTC m=+0.019731429 image pull 425e316ba8ebf74f04aa3640f7deb4fa3f293a78d0c8d097cb679eaf86ddca6c quay.ceph.io/ceph-ci/ceph@sha256:f7ad55f9ccd17ba66aadb19b4d291e67b60ea24e5d8a0c0e92941fd59c81e4ce 2024-09-16T08:23:26.292 INFO:journalctl@ceph.osd.5.smithi144.stdout:Sep 16 08:23:26 smithi144 podman[55052]: 2024-09-16 08:23:26.175841721 +0000 UTC m=+0.134457688 container create 59ec97fb3cf3b1e0948a5d9c63d1c367ba6a57bb7ec3c9822ae00c6637705ab7 (image=quay.ceph.io/ceph-ci/ceph@sha256:f7ad55f9ccd17ba66aadb19b4d291e67b60ea24e5d8a0c0e92941fd59c81e4ce, name=ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-5, ceph=True, RELEASE=wip-guits-main-2024-09-13-1248-8293d73, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, GIT_CLEAN=True) 2024-09-16T08:23:26.832 INFO:journalctl@ceph.osd.5.smithi144.stdout:Sep 16 08:23:26 smithi144 podman[55052]: 2024-09-16 08:23:26.428506592 +0000 UTC m=+0.387122560 container init 59ec97fb3cf3b1e0948a5d9c63d1c367ba6a57bb7ec3c9822ae00c6637705ab7 (image=quay.ceph.io/ceph-ci/ceph@sha256:f7ad55f9ccd17ba66aadb19b4d291e67b60ea24e5d8a0c0e92941fd59c81e4ce, name=ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-5, RELEASE=wip-guits-main-2024-09-13-1248-8293d73, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2) 2024-09-16T08:23:26.832 INFO:journalctl@ceph.osd.5.smithi144.stdout:Sep 16 08:23:26 smithi144 podman[55052]: 2024-09-16 08:23:26.43203521 +0000 UTC m=+0.390651181 container start 59ec97fb3cf3b1e0948a5d9c63d1c367ba6a57bb7ec3c9822ae00c6637705ab7 (image=quay.ceph.io/ceph-ci/ceph@sha256:f7ad55f9ccd17ba66aadb19b4d291e67b60ea24e5d8a0c0e92941fd59c81e4ce, name=ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-5, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, ceph=True, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, RELEASE=wip-guits-main-2024-09-13-1248-8293d73, org.label-schema.license=GPLv2) 2024-09-16T08:23:26.832 INFO:journalctl@ceph.osd.5.smithi144.stdout:Sep 16 08:23:26 smithi144 ceph-osd[55071]: -- 172.21.15.144:0/2007430018 <== mon.1 v2:172.21.15.144:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x559e0c6721e0 con 0x559e0b80bc00 2024-09-16T08:23:26.832 INFO:journalctl@ceph.osd.5.smithi144.stdout:Sep 16 08:23:26 smithi144 bash[55052]: 59ec97fb3cf3b1e0948a5d9c63d1c367ba6a57bb7ec3c9822ae00c6637705ab7 2024-09-16T08:23:26.832 INFO:journalctl@ceph.osd.5.smithi144.stdout:Sep 16 08:23:26 smithi144 systemd[1]: Started Ceph osd.5 for 5622b2fe-7402-11ef-bceb-c7b262605968. 2024-09-16T08:23:27.087 INFO:teuthology.orchestra.run.smithi028.stdout: 5 247810 9.56 MB/sec execute 505 sec latency 39.266 ms 2024-09-16T08:23:27.713 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:23:27 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:23:27.713 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:23:27 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:23:27.713 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:23:27 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-16T08:23:27.713 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:23:27 smithi028 ceph-mon[64115]: pgmap v230: 97 pgs: 9 active+undersized, 22 active+undersized+degraded, 66 active+clean; 337 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 1.7 KiB/s rd, 3.6 MiB/s wr, 399 op/s; 168/1602 objects degraded (10.487%) 2024-09-16T08:23:27.832 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:23:27 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:23:27.832 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:23:27 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:23:27.832 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:23:27 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-16T08:23:27.832 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:23:27 smithi144 ceph-mon[49149]: pgmap v230: 97 pgs: 9 active+undersized, 22 active+undersized+degraded, 66 active+clean; 337 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 1.7 KiB/s rd, 3.6 MiB/s wr, 399 op/s; 168/1602 objects degraded (10.487%) 2024-09-16T08:23:28.087 INFO:teuthology.orchestra.run.smithi028.stdout: 5 248107 9.55 MB/sec execute 506 sec latency 59.127 ms 2024-09-16T08:23:28.179 INFO:journalctl@ceph.osd.5.smithi144.stdout:Sep 16 08:23:28 smithi144 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-5[55067]: 2024-09-16T08:23:28.064+0000 7fdc6e82b640 -1 Falling back to public interface 2024-09-16T08:23:28.728 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:23:28 smithi028 ceph-mon[64115]: Health check update: Degraded data redundancy: 168/1602 objects degraded (10.487%), 22 pgs degraded (PG_DEGRADED) 2024-09-16T08:23:28.831 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:23:28 smithi144 ceph-mon[49149]: Health check update: Degraded data redundancy: 168/1602 objects degraded (10.487%), 22 pgs degraded (PG_DEGRADED) 2024-09-16T08:23:29.087 INFO:teuthology.orchestra.run.smithi028.stdout: 5 248362 9.54 MB/sec execute 507 sec latency 77.182 ms 2024-09-16T08:23:29.764 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:23:29 smithi028 ceph-mon[64115]: pgmap v231: 97 pgs: 9 active+undersized, 22 active+undersized+degraded, 66 active+clean; 340 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 4.1 KiB/s rd, 4.5 MiB/s wr, 508 op/s; 176/1683 objects degraded (10.458%) 2024-09-16T08:23:29.959 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:23:29 smithi144 ceph-mon[49149]: pgmap v231: 97 pgs: 9 active+undersized, 22 active+undersized+degraded, 66 active+clean; 340 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 4.1 KiB/s rd, 4.5 MiB/s wr, 508 op/s; 176/1683 objects degraded (10.458%) 2024-09-16T08:23:30.087 INFO:teuthology.orchestra.run.smithi028.stdout: 5 248565 9.53 MB/sec execute 508 sec latency 54.712 ms 2024-09-16T08:23:31.088 INFO:teuthology.orchestra.run.smithi028.stdout: 5 248828 9.53 MB/sec execute 509 sec latency 75.789 ms 2024-09-16T08:23:31.307 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:23:31 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:23:31.308 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:23:31 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:23:31.339 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:23:31 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:23:31.339 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:23:31 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:23:31.339 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:23:31 smithi028 ceph-mon[64115]: pgmap v232: 97 pgs: 9 active+undersized, 22 active+undersized+degraded, 66 active+clean; 341 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 3.7 KiB/s rd, 3.9 MiB/s wr, 337 op/s; 179/1686 objects degraded (10.617%) 2024-09-16T08:23:31.339 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:23:31 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:23:31.339 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:23:31 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:23:31.392 INFO:teuthology.orchestra.run.smithi028.stdout:true 2024-09-16T08:23:31.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:23:31 smithi144 ceph-mon[49149]: pgmap v232: 97 pgs: 9 active+undersized, 22 active+undersized+degraded, 66 active+clean; 341 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 3.7 KiB/s rd, 3.9 MiB/s wr, 337 op/s; 179/1686 objects degraded (10.617%) 2024-09-16T08:23:31.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:23:31 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:23:31.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:23:31 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:23:32.087 INFO:teuthology.orchestra.run.smithi028.stdout: 5 249062 9.51 MB/sec execute 510 sec latency 77.153 ms 2024-09-16T08:23:32.362 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:23:32 smithi028 ceph-mon[64115]: from='client.34378 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T08:23:32.362 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:23:32 smithi028 ceph-mon[64115]: from='client.44321 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T08:23:32.362 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:23:32 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:23:32.363 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:23:32 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-16T08:23:32.493 INFO:teuthology.orchestra.run.smithi028.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-16T08:23:32.493 INFO:teuthology.orchestra.run.smithi028.stdout:alertmanager.smithi028 smithi028 *:9093,9094 running (7m) 80s ago 16m 18.6M - 0.27.0 11f11916f8cd a14fcb4cb2b6 2024-09-16T08:23:32.493 INFO:teuthology.orchestra.run.smithi028.stdout:crash.smithi028 smithi028 running (4m) 80s ago 16m 6899k - 19.3.0-4917-g8293d73f 425e316ba8eb 4bb89df2a867 2024-09-16T08:23:32.493 INFO:teuthology.orchestra.run.smithi028.stdout:crash.smithi144 smithi144 running (4m) 2s ago 16m 6895k - 19.3.0-4917-g8293d73f 425e316ba8eb 24a96875d7af 2024-09-16T08:23:32.493 INFO:teuthology.orchestra.run.smithi028.stdout:grafana.smithi028 smithi028 *:3000 running (7m) 80s ago 16m 45.9M - dad864ee21e9 b55ba8b944a1 2024-09-16T08:23:32.493 INFO:teuthology.orchestra.run.smithi028.stdout:mds.foofs.smithi028.srtgev smithi028 running (11m) 80s ago 11m 38.5M - 17.2.0 e1d6a67b021e 250e7a7f754c 2024-09-16T08:23:32.494 INFO:teuthology.orchestra.run.smithi028.stdout:mds.foofs.smithi144.oozgsi smithi144 running (11m) 2s ago 11m 17.5M - 17.2.0 e1d6a67b021e ecf4c2bb0fdc 2024-09-16T08:23:32.494 INFO:teuthology.orchestra.run.smithi028.stdout:mgr.smithi028.azvyzy smithi028 *:8443,9283,8765 running (6m) 80s ago 18m 519M - 19.3.0-4917-g8293d73f 425e316ba8eb e01d4382d03f 2024-09-16T08:23:32.494 INFO:teuthology.orchestra.run.smithi028.stdout:mgr.smithi144.utzszj smithi144 *:8443,9283,8765 running (6m) 2s ago 16m 457M - 19.3.0-4917-g8293d73f 425e316ba8eb 71f7f802bffb 2024-09-16T08:23:32.494 INFO:teuthology.orchestra.run.smithi028.stdout:mon.smithi028 smithi028 running (5m) 80s ago 18m 49.8M 2048M 19.3.0-4917-g8293d73f 425e316ba8eb 14efcedc0088 2024-09-16T08:23:32.494 INFO:teuthology.orchestra.run.smithi028.stdout:mon.smithi144 smithi144 running (5m) 2s ago 16m 46.5M 2048M 19.3.0-4917-g8293d73f 425e316ba8eb c081c5ed44b5 2024-09-16T08:23:32.494 INFO:teuthology.orchestra.run.smithi028.stdout:nfs.foo.0.0.smithi028.dkttgj smithi028 *:2049 running (10m) 80s ago 10m 285M - 4.0 e1d6a67b021e b5dccc45a6ed 2024-09-16T08:23:32.494 INFO:teuthology.orchestra.run.smithi028.stdout:nfs.foo.1.0.smithi144.epipkg smithi144 *:2049 running (10m) 2s ago 10m 87.4M - 4.0 e1d6a67b021e faa89e61f50e 2024-09-16T08:23:32.494 INFO:teuthology.orchestra.run.smithi028.stdout:node-exporter.smithi028 smithi028 *:9100 running (7m) 80s ago 16m 9.92M - 1.7.0 72c9c2088986 9e2b8e8a63a9 2024-09-16T08:23:32.494 INFO:teuthology.orchestra.run.smithi028.stdout:node-exporter.smithi144 smithi144 *:9100 running (7m) 2s ago 16m 10.8M - 1.7.0 72c9c2088986 78834c5e992f 2024-09-16T08:23:32.494 INFO:teuthology.orchestra.run.smithi028.stdout:osd.0 smithi028 running (3m) 80s ago 15m 354M 1937M 19.3.0-4917-g8293d73f 425e316ba8eb 2fc02ba4f14f 2024-09-16T08:23:32.494 INFO:teuthology.orchestra.run.smithi028.stdout:osd.1 smithi028 running (3m) 80s ago 15m 266M 1937M 19.3.0-4917-g8293d73f 425e316ba8eb df34bd603d8f 2024-09-16T08:23:32.494 INFO:teuthology.orchestra.run.smithi028.stdout:osd.2 smithi028 running (2m) 80s ago 14m 220M 1937M 19.3.0-4917-g8293d73f 425e316ba8eb c92be6327b12 2024-09-16T08:23:32.495 INFO:teuthology.orchestra.run.smithi028.stdout:osd.3 smithi028 running (89s) 80s ago 14m 11.2M 1937M 19.3.0-4917-g8293d73f 425e316ba8eb bd84ba1224f2 2024-09-16T08:23:32.495 INFO:teuthology.orchestra.run.smithi028.stdout:osd.4 smithi144 running (44s) 2s ago 13m 270M 2704M 19.3.0-4917-g8293d73f 425e316ba8eb c48cb68e901e 2024-09-16T08:23:32.495 INFO:teuthology.orchestra.run.smithi028.stdout:osd.5 smithi144 running (6s) 2s ago 13m 12.1M 2704M 19.3.0-4917-g8293d73f 425e316ba8eb 59ec97fb3cf3 2024-09-16T08:23:32.495 INFO:teuthology.orchestra.run.smithi028.stdout:osd.6 smithi144 running (12m) 2s ago 12m 509M 2704M 17.2.0 e1d6a67b021e 7f9981be792f 2024-09-16T08:23:32.495 INFO:teuthology.orchestra.run.smithi028.stdout:osd.7 smithi144 running (12m) 2s ago 12m 784M 2704M 17.2.0 e1d6a67b021e dada74ddbe4c 2024-09-16T08:23:32.495 INFO:teuthology.orchestra.run.smithi028.stdout:prometheus.smithi028 smithi028 *:9095 running (6m) 80s ago 15m 55.3M - 2.51.0 1d3b7f56885b 42667809df2b 2024-09-16T08:23:32.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:23:32 smithi144 ceph-mon[49149]: from='client.34378 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T08:23:32.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:23:32 smithi144 ceph-mon[49149]: from='client.44321 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T08:23:32.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:23:32 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:23:32.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:23:32 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-16T08:23:33.087 INFO:teuthology.orchestra.run.smithi028.stdout: 5 249427 9.51 MB/sec execute 511 sec latency 69.555 ms 2024-09-16T08:23:33.243 INFO:teuthology.orchestra.run.smithi028.stdout:{ 2024-09-16T08:23:33.243 INFO:teuthology.orchestra.run.smithi028.stdout: "mon": { 2024-09-16T08:23:33.243 INFO:teuthology.orchestra.run.smithi028.stdout: "ceph version 19.3.0-4917-g8293d73f (8293d73f8690540e843a81caec373f9cc29cf705) squid (dev)": 2 2024-09-16T08:23:33.243 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-09-16T08:23:33.243 INFO:teuthology.orchestra.run.smithi028.stdout: "mgr": { 2024-09-16T08:23:33.243 INFO:teuthology.orchestra.run.smithi028.stdout: "ceph version 19.3.0-4917-g8293d73f (8293d73f8690540e843a81caec373f9cc29cf705) squid (dev)": 2 2024-09-16T08:23:33.244 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-09-16T08:23:33.244 INFO:teuthology.orchestra.run.smithi028.stdout: "osd": { 2024-09-16T08:23:33.244 INFO:teuthology.orchestra.run.smithi028.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2, 2024-09-16T08:23:33.244 INFO:teuthology.orchestra.run.smithi028.stdout: "ceph version 19.3.0-4917-g8293d73f (8293d73f8690540e843a81caec373f9cc29cf705) squid (dev)": 5 2024-09-16T08:23:33.244 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-09-16T08:23:33.244 INFO:teuthology.orchestra.run.smithi028.stdout: "mds": { 2024-09-16T08:23:33.244 INFO:teuthology.orchestra.run.smithi028.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-16T08:23:33.244 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-09-16T08:23:33.244 INFO:teuthology.orchestra.run.smithi028.stdout: "overall": { 2024-09-16T08:23:33.244 INFO:teuthology.orchestra.run.smithi028.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 4, 2024-09-16T08:23:33.244 INFO:teuthology.orchestra.run.smithi028.stdout: "ceph version 19.3.0-4917-g8293d73f (8293d73f8690540e843a81caec373f9cc29cf705) squid (dev)": 9 2024-09-16T08:23:33.244 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-09-16T08:23:33.244 INFO:teuthology.orchestra.run.smithi028.stdout:} 2024-09-16T08:23:33.366 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:23:33 smithi028 ceph-mon[64115]: from='client.44325 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T08:23:33.366 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:23:33 smithi028 ceph-mon[64115]: pgmap v233: 97 pgs: 9 active+undersized, 22 active+undersized+degraded, 66 active+clean; 340 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.8 KiB/s rd, 4.0 MiB/s wr, 404 op/s; 178/1653 objects degraded (10.768%) 2024-09-16T08:23:33.366 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:23:33 smithi028 ceph-mon[64115]: from='client.? 172.21.15.28:0/4031564849' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-16T08:23:33.609 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:23:33 smithi144 ceph-mon[49149]: from='client.44325 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T08:23:33.610 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:23:33 smithi144 ceph-mon[49149]: pgmap v233: 97 pgs: 9 active+undersized, 22 active+undersized+degraded, 66 active+clean; 340 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.8 KiB/s rd, 4.0 MiB/s wr, 404 op/s; 178/1653 objects degraded (10.768%) 2024-09-16T08:23:33.610 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:23:33 smithi144 ceph-mon[49149]: from='client.? 172.21.15.28:0/4031564849' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-16T08:23:33.834 INFO:teuthology.orchestra.run.smithi028.stdout:{ 2024-09-16T08:23:33.834 INFO:teuthology.orchestra.run.smithi028.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:f7ad55f9ccd17ba66aadb19b4d291e67b60ea24e5d8a0c0e92941fd59c81e4ce", 2024-09-16T08:23:33.835 INFO:teuthology.orchestra.run.smithi028.stdout: "in_progress": true, 2024-09-16T08:23:33.835 INFO:teuthology.orchestra.run.smithi028.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-16T08:23:33.835 INFO:teuthology.orchestra.run.smithi028.stdout: "services_complete": [ 2024-09-16T08:23:33.835 INFO:teuthology.orchestra.run.smithi028.stdout: "crash", 2024-09-16T08:23:33.835 INFO:teuthology.orchestra.run.smithi028.stdout: "mon", 2024-09-16T08:23:33.835 INFO:teuthology.orchestra.run.smithi028.stdout: "mgr" 2024-09-16T08:23:33.835 INFO:teuthology.orchestra.run.smithi028.stdout: ], 2024-09-16T08:23:33.835 INFO:teuthology.orchestra.run.smithi028.stdout: "progress": "12/23 daemons upgraded", 2024-09-16T08:23:33.835 INFO:teuthology.orchestra.run.smithi028.stdout: "message": "Currently upgrading osd daemons", 2024-09-16T08:23:33.835 INFO:teuthology.orchestra.run.smithi028.stdout: "is_paused": false 2024-09-16T08:23:33.835 INFO:teuthology.orchestra.run.smithi028.stdout:} 2024-09-16T08:23:34.088 INFO:teuthology.orchestra.run.smithi028.stdout: 5 249678 9.50 MB/sec execute 512 sec latency 57.303 ms 2024-09-16T08:23:34.393 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:23:34 smithi028 ceph-mon[64115]: Health check update: Degraded data redundancy: 178/1653 objects degraded (10.768%), 22 pgs degraded (PG_DEGRADED) 2024-09-16T08:23:34.393 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:23:34 smithi028 ceph-mon[64115]: from='client.44333 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T08:23:34.556 INFO:teuthology.orchestra.run.smithi028.stdout:HEALTH_WARN 1 osds down; Degraded data redundancy: 178/1653 objects degraded (10.768%), 22 pgs degraded 2024-09-16T08:23:34.556 INFO:teuthology.orchestra.run.smithi028.stdout:[WRN] OSD_DOWN: 1 osds down 2024-09-16T08:23:34.556 INFO:teuthology.orchestra.run.smithi028.stdout: osd.5 (root=default,host=smithi144) is down 2024-09-16T08:23:34.556 INFO:teuthology.orchestra.run.smithi028.stdout:[WRN] PG_DEGRADED: Degraded data redundancy: 178/1653 objects degraded (10.768%), 22 pgs degraded 2024-09-16T08:23:34.557 INFO:teuthology.orchestra.run.smithi028.stdout: pg 2.2 is active+undersized+degraded, acting [1,6] 2024-09-16T08:23:34.557 INFO:teuthology.orchestra.run.smithi028.stdout: pg 2.3 is active+undersized+degraded, acting [2,7] 2024-09-16T08:23:34.557 INFO:teuthology.orchestra.run.smithi028.stdout: pg 2.b is active+undersized+degraded, acting [7,4] 2024-09-16T08:23:34.557 INFO:teuthology.orchestra.run.smithi028.stdout: pg 2.c is active+undersized+degraded, acting [2,0] 2024-09-16T08:23:34.557 INFO:teuthology.orchestra.run.smithi028.stdout: pg 2.12 is active+undersized+degraded, acting [3,7] 2024-09-16T08:23:34.557 INFO:teuthology.orchestra.run.smithi028.stdout: pg 2.14 is active+undersized+degraded, acting [6,3] 2024-09-16T08:23:34.557 INFO:teuthology.orchestra.run.smithi028.stdout: pg 2.15 is active+undersized+degraded, acting [1,0] 2024-09-16T08:23:34.557 INFO:teuthology.orchestra.run.smithi028.stdout: pg 2.16 is active+undersized+degraded, acting [6,2] 2024-09-16T08:23:34.557 INFO:teuthology.orchestra.run.smithi028.stdout: pg 2.17 is active+undersized+degraded, acting [6,2] 2024-09-16T08:23:34.557 INFO:teuthology.orchestra.run.smithi028.stdout: pg 2.18 is active+undersized+degraded, acting [4,7] 2024-09-16T08:23:34.557 INFO:teuthology.orchestra.run.smithi028.stdout: pg 2.1c is active+undersized+degraded, acting [7,2] 2024-09-16T08:23:34.557 INFO:teuthology.orchestra.run.smithi028.stdout: pg 3.2 is active+undersized+degraded, acting [3,6] 2024-09-16T08:23:34.558 INFO:teuthology.orchestra.run.smithi028.stdout: pg 3.4 is active+undersized+degraded, acting [1,2] 2024-09-16T08:23:34.558 INFO:teuthology.orchestra.run.smithi028.stdout: pg 3.5 is active+undersized+degraded, acting [3,2] 2024-09-16T08:23:34.558 INFO:teuthology.orchestra.run.smithi028.stdout: pg 3.c is active+undersized+degraded, acting [3,6] 2024-09-16T08:23:34.558 INFO:teuthology.orchestra.run.smithi028.stdout: pg 3.d is active+undersized+degraded, acting [7,6] 2024-09-16T08:23:34.558 INFO:teuthology.orchestra.run.smithi028.stdout: pg 3.10 is active+undersized+degraded, acting [6,0] 2024-09-16T08:23:34.558 INFO:teuthology.orchestra.run.smithi028.stdout: pg 3.16 is active+undersized+degraded, acting [7,1] 2024-09-16T08:23:34.558 INFO:teuthology.orchestra.run.smithi028.stdout: pg 3.17 is active+undersized+degraded, acting [0,3] 2024-09-16T08:23:34.558 INFO:teuthology.orchestra.run.smithi028.stdout: pg 3.1c is active+undersized+degraded, acting [4,1] 2024-09-16T08:23:34.558 INFO:teuthology.orchestra.run.smithi028.stdout: pg 3.1d is active+undersized+degraded, acting [4,6] 2024-09-16T08:23:34.558 INFO:teuthology.orchestra.run.smithi028.stdout: pg 3.1f is active+undersized+degraded, acting [0,2] 2024-09-16T08:23:34.600 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:23:34 smithi144 ceph-mon[49149]: Health check update: Degraded data redundancy: 178/1653 objects degraded (10.768%), 22 pgs degraded (PG_DEGRADED) 2024-09-16T08:23:34.600 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:23:34 smithi144 ceph-mon[49149]: from='client.44333 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T08:23:35.088 INFO:teuthology.orchestra.run.smithi028.stdout: 5 249937 9.50 MB/sec execute 513 sec latency 61.855 ms 2024-09-16T08:23:35.426 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:23:35 smithi028 ceph-mon[64115]: from='client.? 172.21.15.28:0/554210839' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-16T08:23:35.426 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:23:35 smithi028 ceph-mon[64115]: pgmap v234: 97 pgs: 9 active+undersized, 22 active+undersized+degraded, 66 active+clean; 337 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 4.4 MiB/s wr, 420 op/s; 171/1626 objects degraded (10.517%) 2024-09-16T08:23:35.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:23:35 smithi144 ceph-mon[49149]: from='client.? 172.21.15.28:0/554210839' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-16T08:23:35.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:23:35 smithi144 ceph-mon[49149]: pgmap v234: 97 pgs: 9 active+undersized, 22 active+undersized+degraded, 66 active+clean; 337 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 4.4 MiB/s wr, 420 op/s; 171/1626 objects degraded (10.517%) 2024-09-16T08:23:35.582 INFO:journalctl@ceph.osd.5.smithi144.stdout:Sep 16 08:23:35 smithi144 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-5[55067]: 2024-09-16T08:23:35.466+0000 7fdc6e82b640 -1 osd.5 0 read_superblock omap replica is missing. 2024-09-16T08:23:36.088 INFO:teuthology.orchestra.run.smithi028.stdout: 5 250207 9.50 MB/sec execute 514 sec latency 39.551 ms 2024-09-16T08:23:36.832 INFO:journalctl@ceph.osd.5.smithi144.stdout:Sep 16 08:23:36 smithi144 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-5[55067]: 2024-09-16T08:23:36.577+0000 7fdc6e82b640 -1 osd.5 127 log_to_monitors true 2024-09-16T08:23:36.832 INFO:journalctl@ceph.osd.5.smithi144.stdout:Sep 16 08:23:36 smithi144 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-5[55067]: 2024-09-16T08:23:36.815+0000 7fdc665d5640 -1 osd.5 127 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-16T08:23:36.832 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:23:36 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:23:36.832 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:23:36 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:23:36.832 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:23:36 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-16T08:23:36.832 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:23:36 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-16T08:23:36.833 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:23:36 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:23:36.833 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:23:36 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:23:36.833 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:23:36 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-16T08:23:36.833 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:23:36 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "versions"} : dispatch 2024-09-16T08:23:36.833 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:23:36 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "versions"} : dispatch 2024-09-16T08:23:36.833 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:23:36 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "versions"} : dispatch 2024-09-16T08:23:36.833 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:23:36 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16} : dispatch 2024-09-16T08:23:36.833 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:23:36 smithi144 ceph-mon[49149]: from='osd.5 [v2:172.21.15.144:6808/2578680151,v1:172.21.15.144:6809/2578680151]' entity='osd.5' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]} : dispatch 2024-09-16T08:23:36.833 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:23:36 smithi144 ceph-mon[49149]: from='osd.5 ' entity='osd.5' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]} : dispatch 2024-09-16T08:23:36.969 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:23:36 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:23:36.969 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:23:36 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:23:36.969 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:23:36 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-16T08:23:36.969 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:23:36 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-16T08:23:36.969 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:23:36 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:23:36.970 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:23:36 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:23:36.970 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:23:36 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-16T08:23:36.970 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:23:36 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "versions"} : dispatch 2024-09-16T08:23:36.970 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:23:36 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "versions"} : dispatch 2024-09-16T08:23:36.970 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:23:36 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "versions"} : dispatch 2024-09-16T08:23:36.970 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:23:36 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16} : dispatch 2024-09-16T08:23:36.970 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:23:36 smithi028 ceph-mon[64115]: from='osd.5 [v2:172.21.15.144:6808/2578680151,v1:172.21.15.144:6809/2578680151]' entity='osd.5' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]} : dispatch 2024-09-16T08:23:36.970 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:23:36 smithi028 ceph-mon[64115]: from='osd.5 ' entity='osd.5' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]} : dispatch 2024-09-16T08:23:37.088 INFO:teuthology.orchestra.run.smithi028.stdout: 5 250573 9.50 MB/sec execute 515 sec latency 72.801 ms 2024-09-16T08:23:38.003 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:23:37 smithi028 ceph-mon[64115]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-09-16T08:23:38.003 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:23:37 smithi028 ceph-mon[64115]: Upgrade: unsafe to stop osd(s) at this time (12 PGs are or would become offline) 2024-09-16T08:23:38.003 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:23:37 smithi028 ceph-mon[64115]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-09-16T08:23:38.003 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:23:37 smithi028 ceph-mon[64115]: osdmap e130: 8 total, 7 up, 8 in 2024-09-16T08:23:38.003 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:23:37 smithi028 ceph-mon[64115]: from='osd.5 [v2:172.21.15.144:6808/2578680151,v1:172.21.15.144:6809/2578680151]' entity='osd.5' cmd={"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi144", "root=default"]} : dispatch 2024-09-16T08:23:38.004 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:23:37 smithi028 ceph-mon[64115]: from='osd.5 ' entity='osd.5' cmd={"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi144", "root=default"]} : dispatch 2024-09-16T08:23:38.004 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:23:37 smithi028 ceph-mon[64115]: pgmap v236: 97 pgs: 9 active+undersized, 22 active+undersized+degraded, 66 active+clean; 335 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 6.9 KiB/s rd, 4.0 MiB/s wr, 436 op/s; 170/1605 objects degraded (10.592%) 2024-09-16T08:23:38.082 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:23:37 smithi144 ceph-mon[49149]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-09-16T08:23:38.082 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:23:37 smithi144 ceph-mon[49149]: Upgrade: unsafe to stop osd(s) at this time (12 PGs are or would become offline) 2024-09-16T08:23:38.082 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:23:37 smithi144 ceph-mon[49149]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-09-16T08:23:38.082 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:23:37 smithi144 ceph-mon[49149]: osdmap e130: 8 total, 7 up, 8 in 2024-09-16T08:23:38.082 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:23:37 smithi144 ceph-mon[49149]: from='osd.5 [v2:172.21.15.144:6808/2578680151,v1:172.21.15.144:6809/2578680151]' entity='osd.5' cmd={"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi144", "root=default"]} : dispatch 2024-09-16T08:23:38.082 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:23:37 smithi144 ceph-mon[49149]: from='osd.5 ' entity='osd.5' cmd={"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi144", "root=default"]} : dispatch 2024-09-16T08:23:38.082 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:23:37 smithi144 ceph-mon[49149]: pgmap v236: 97 pgs: 9 active+undersized, 22 active+undersized+degraded, 66 active+clean; 335 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 6.9 KiB/s rd, 4.0 MiB/s wr, 436 op/s; 170/1605 objects degraded (10.592%) 2024-09-16T08:23:38.088 INFO:teuthology.orchestra.run.smithi028.stdout: 5 250862 9.51 MB/sec execute 516 sec latency 284.066 ms 2024-09-16T08:23:39.081 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:23:38 smithi028 ceph-mon[64115]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-16T08:23:39.081 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:23:38 smithi028 ceph-mon[64115]: osd.5 [v2:172.21.15.144:6808/2578680151,v1:172.21.15.144:6809/2578680151] boot 2024-09-16T08:23:39.081 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:23:38 smithi028 ceph-mon[64115]: osdmap e131: 8 total, 8 up, 8 in 2024-09-16T08:23:39.081 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:23:38 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "osd metadata", "id": 5} : dispatch 2024-09-16T08:23:39.082 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:23:38 smithi144 ceph-mon[49149]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-16T08:23:39.082 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:23:38 smithi144 ceph-mon[49149]: osd.5 [v2:172.21.15.144:6808/2578680151,v1:172.21.15.144:6809/2578680151] boot 2024-09-16T08:23:39.082 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:23:38 smithi144 ceph-mon[49149]: osdmap e131: 8 total, 8 up, 8 in 2024-09-16T08:23:39.082 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:23:38 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "osd metadata", "id": 5} : dispatch 2024-09-16T08:23:39.088 INFO:teuthology.orchestra.run.smithi028.stdout: 5 250957 9.49 MB/sec execute 517 sec latency 1072.925 ms 2024-09-16T08:23:39.845 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:23:39 smithi028 ceph-mon[64115]: osdmap e132: 8 total, 8 up, 8 in 2024-09-16T08:23:39.845 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:23:39 smithi028 ceph-mon[64115]: pgmap v239: 97 pgs: 8 peering, 7 active+undersized, 16 active+undersized+degraded, 66 active+clean; 338 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 2.3 KiB/s rd, 3.2 MiB/s wr, 248 op/s; 128/1653 objects degraded (7.743%) 2024-09-16T08:23:39.845 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:23:39 smithi028 ceph-mon[64115]: osdmap e133: 8 total, 8 up, 8 in 2024-09-16T08:23:40.081 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:23:39 smithi144 ceph-mon[49149]: osdmap e132: 8 total, 8 up, 8 in 2024-09-16T08:23:40.082 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:23:39 smithi144 ceph-mon[49149]: pgmap v239: 97 pgs: 8 peering, 7 active+undersized, 16 active+undersized+degraded, 66 active+clean; 338 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 2.3 KiB/s rd, 3.2 MiB/s wr, 248 op/s; 128/1653 objects degraded (7.743%) 2024-09-16T08:23:40.082 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:23:39 smithi144 ceph-mon[49149]: osdmap e133: 8 total, 8 up, 8 in 2024-09-16T08:23:40.088 INFO:teuthology.orchestra.run.smithi028.stdout: 5 251084 9.48 MB/sec execute 518 sec latency 1471.082 ms 2024-09-16T08:23:40.865 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:23:40 smithi028 ceph-mon[64115]: Health check update: Degraded data redundancy: 128/1653 objects degraded (7.743%), 16 pgs degraded (PG_DEGRADED) 2024-09-16T08:23:40.865 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:23:40 smithi028 ceph-mon[64115]: osdmap e134: 8 total, 8 up, 8 in 2024-09-16T08:23:41.081 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:23:40 smithi144 ceph-mon[49149]: Health check update: Degraded data redundancy: 128/1653 objects degraded (7.743%), 16 pgs degraded (PG_DEGRADED) 2024-09-16T08:23:41.082 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:23:40 smithi144 ceph-mon[49149]: osdmap e134: 8 total, 8 up, 8 in 2024-09-16T08:23:41.088 INFO:teuthology.orchestra.run.smithi028.stdout: 5 251312 9.47 MB/sec execute 519 sec latency 124.103 ms 2024-09-16T08:23:42.088 INFO:teuthology.orchestra.run.smithi028.stdout: 5 251478 9.46 MB/sec execute 520 sec latency 1021.626 ms 2024-09-16T08:23:42.155 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:23:41 smithi028 ceph-mon[64115]: pgmap v242: 97 pgs: 1 activating+undersized+remapped, 8 peering, 5 active+undersized, 12 active+undersized+degraded, 71 active+clean; 344 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 4.0 KiB/s rd, 4.5 MiB/s wr, 414 op/s; 116/1656 objects degraded (7.005%); 363 KiB/s, 8 objects/s recovering 2024-09-16T08:23:42.155 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:23:41 smithi028 ceph-mon[64115]: osdmap e135: 8 total, 8 up, 8 in 2024-09-16T08:23:42.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:23:41 smithi144 ceph-mon[49149]: pgmap v242: 97 pgs: 1 activating+undersized+remapped, 8 peering, 5 active+undersized, 12 active+undersized+degraded, 71 active+clean; 344 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 4.0 KiB/s rd, 4.5 MiB/s wr, 414 op/s; 116/1656 objects degraded (7.005%); 363 KiB/s, 8 objects/s recovering 2024-09-16T08:23:42.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:23:41 smithi144 ceph-mon[49149]: osdmap e135: 8 total, 8 up, 8 in 2024-09-16T08:23:43.088 INFO:teuthology.orchestra.run.smithi028.stdout: 5 251761 9.46 MB/sec execute 521 sec latency 39.158 ms 2024-09-16T08:23:43.176 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:23:42 smithi028 ceph-mon[64115]: osdmap e136: 8 total, 8 up, 8 in 2024-09-16T08:23:43.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:23:42 smithi144 ceph-mon[49149]: osdmap e136: 8 total, 8 up, 8 in 2024-09-16T08:23:44.088 INFO:teuthology.orchestra.run.smithi028.stdout: 5 252135 9.46 MB/sec execute 522 sec latency 42.350 ms 2024-09-16T08:23:44.191 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:23:43 smithi028 ceph-mon[64115]: pgmap v245: 97 pgs: 1 activating+undersized+remapped, 8 peering, 88 active+clean; 345 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 511 B/s rd, 3.4 MiB/s wr, 367 op/s; 23/1671 objects degraded (1.376%); 795 KiB/s, 38 objects/s recovering 2024-09-16T08:23:44.191 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:23:43 smithi028 ceph-mon[64115]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 116/1656 objects degraded (7.005%), 12 pgs degraded) 2024-09-16T08:23:44.191 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:23:43 smithi028 ceph-mon[64115]: Cluster is now healthy 2024-09-16T08:23:44.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:23:43 smithi144 ceph-mon[49149]: pgmap v245: 97 pgs: 1 activating+undersized+remapped, 8 peering, 88 active+clean; 345 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 511 B/s rd, 3.4 MiB/s wr, 367 op/s; 23/1671 objects degraded (1.376%); 795 KiB/s, 38 objects/s recovering 2024-09-16T08:23:44.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:23:43 smithi144 ceph-mon[49149]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 116/1656 objects degraded (7.005%), 12 pgs degraded) 2024-09-16T08:23:44.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:23:43 smithi144 ceph-mon[49149]: Cluster is now healthy 2024-09-16T08:23:45.088 INFO:teuthology.orchestra.run.smithi028.stdout: 5 252569 9.46 MB/sec execute 523 sec latency 54.495 ms 2024-09-16T08:23:45.965 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:23:45 smithi028 ceph-mon[64115]: pgmap v246: 97 pgs: 97 active+clean; 345 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 6.5 KiB/s rd, 4.3 MiB/s wr, 335 op/s; 2.4 MiB/s, 42 objects/s recovering 2024-09-16T08:23:46.088 INFO:teuthology.orchestra.run.smithi028.stdout: 5 252894 9.45 MB/sec execute 524 sec latency 47.409 ms 2024-09-16T08:23:46.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:23:45 smithi144 ceph-mon[49149]: pgmap v246: 97 pgs: 97 active+clean; 345 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 6.5 KiB/s rd, 4.3 MiB/s wr, 335 op/s; 2.4 MiB/s, 42 objects/s recovering 2024-09-16T08:23:47.088 INFO:teuthology.orchestra.run.smithi028.stdout: 5 253226 9.46 MB/sec execute 525 sec latency 68.678 ms 2024-09-16T08:23:47.993 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:23:47 smithi028 ceph-mon[64115]: pgmap v247: 97 pgs: 97 active+clean; 344 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 4.4 KiB/s rd, 3.5 MiB/s wr, 269 op/s; 1.7 MiB/s, 30 objects/s recovering 2024-09-16T08:23:47.993 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:23:47 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:23:47.993 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:23:47 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-16T08:23:48.088 INFO:teuthology.orchestra.run.smithi028.stdout: 5 253644 9.46 MB/sec execute 526 sec latency 40.602 ms 2024-09-16T08:23:48.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:23:47 smithi144 ceph-mon[49149]: pgmap v247: 97 pgs: 97 active+clean; 344 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 4.4 KiB/s rd, 3.5 MiB/s wr, 269 op/s; 1.7 MiB/s, 30 objects/s recovering 2024-09-16T08:23:48.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:23:47 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:23:48.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:23:47 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-16T08:23:49.088 INFO:teuthology.orchestra.run.smithi028.stdout: 5 253944 9.46 MB/sec execute 527 sec latency 72.579 ms 2024-09-16T08:23:50.054 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:23:49 smithi028 ceph-mon[64115]: pgmap v248: 97 pgs: 97 active+clean; 340 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 8.1 KiB/s rd, 4.3 MiB/s wr, 468 op/s; 1.3 MiB/s, 22 objects/s recovering 2024-09-16T08:23:50.088 INFO:teuthology.orchestra.run.smithi028.stdout: 5 254263 9.47 MB/sec execute 528 sec latency 44.945 ms 2024-09-16T08:23:50.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:23:49 smithi144 ceph-mon[49149]: pgmap v248: 97 pgs: 97 active+clean; 340 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 8.1 KiB/s rd, 4.3 MiB/s wr, 468 op/s; 1.3 MiB/s, 22 objects/s recovering 2024-09-16T08:23:51.088 INFO:teuthology.orchestra.run.smithi028.stdout: 5 254502 9.46 MB/sec execute 529 sec latency 59.558 ms 2024-09-16T08:23:52.088 INFO:teuthology.orchestra.run.smithi028.stdout: 5 254772 9.45 MB/sec execute 530 sec latency 88.913 ms 2024-09-16T08:23:52.110 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:23:51 smithi028 ceph-mon[64115]: pgmap v249: 97 pgs: 97 active+clean; 341 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.2 MiB/s wr, 414 op/s; 1.1 MiB/s, 17 objects/s recovering 2024-09-16T08:23:52.110 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:23:51 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16} : dispatch 2024-09-16T08:23:52.143 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:23:51 smithi144 ceph-mon[49149]: pgmap v249: 97 pgs: 97 active+clean; 341 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.2 MiB/s wr, 414 op/s; 1.1 MiB/s, 17 objects/s recovering 2024-09-16T08:23:52.143 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:23:51 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16} : dispatch 2024-09-16T08:23:53.089 INFO:teuthology.orchestra.run.smithi028.stdout: 5 254955 9.44 MB/sec execute 531 sec latency 61.199 ms 2024-09-16T08:23:53.120 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:23:52 smithi028 ceph-mon[64115]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-09-16T08:23:53.120 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:23:52 smithi028 ceph-mon[64115]: Upgrade: osd.6 is safe to restart 2024-09-16T08:23:53.120 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:23:52 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:23:53.245 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:23:52 smithi144 ceph-mon[49149]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-09-16T08:23:53.245 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:23:52 smithi144 ceph-mon[49149]: Upgrade: osd.6 is safe to restart 2024-09-16T08:23:53.245 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:23:52 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:23:54.088 INFO:teuthology.orchestra.run.smithi028.stdout: 5 255135 9.43 MB/sec execute 532 sec latency 68.980 ms 2024-09-16T08:23:54.156 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:23:53 smithi028 ceph-mon[64115]: pgmap v250: 97 pgs: 97 active+clean; 344 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 6.1 KiB/s rd, 4.3 MiB/s wr, 420 op/s; 860 KiB/s, 7 objects/s recovering 2024-09-16T08:23:54.156 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:23:53 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:23:54.156 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:23:53 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "auth get", "entity": "osd.6"} : dispatch 2024-09-16T08:23:54.156 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:23:53 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-16T08:23:54.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:23:53 smithi144 ceph-mon[49149]: pgmap v250: 97 pgs: 97 active+clean; 344 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 6.1 KiB/s rd, 4.3 MiB/s wr, 420 op/s; 860 KiB/s, 7 objects/s recovering 2024-09-16T08:23:54.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:23:53 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:23:54.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:23:53 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "auth get", "entity": "osd.6"} : dispatch 2024-09-16T08:23:54.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:23:53 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-16T08:23:55.089 INFO:teuthology.orchestra.run.smithi028.stdout: 5 255348 9.42 MB/sec execute 533 sec latency 71.945 ms 2024-09-16T08:23:55.213 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:23:54 smithi028 ceph-mon[64115]: Upgrade: Updating osd.6 2024-09-16T08:23:55.213 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:23:54 smithi028 ceph-mon[64115]: Deploying daemon osd.6 on smithi144 2024-09-16T08:23:55.250 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:23:54 smithi144 ceph-mon[49149]: Upgrade: Updating osd.6 2024-09-16T08:23:55.250 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:23:54 smithi144 ceph-mon[49149]: Deploying daemon osd.6 on smithi144 2024-09-16T08:23:56.001 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:23:55 smithi028 ceph-mon[64115]: pgmap v251: 97 pgs: 97 active+clean; 345 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 8.0 KiB/s rd, 4.9 MiB/s wr, 494 op/s; 757 KiB/s, 4 objects/s recovering 2024-09-16T08:23:56.088 INFO:teuthology.orchestra.run.smithi028.stdout: 5 255618 9.42 MB/sec execute 534 sec latency 69.326 ms 2024-09-16T08:23:56.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:23:55 smithi144 ceph-mon[49149]: pgmap v251: 97 pgs: 97 active+clean; 345 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 8.0 KiB/s rd, 4.9 MiB/s wr, 494 op/s; 757 KiB/s, 4 objects/s recovering 2024-09-16T08:23:56.738 INFO:journalctl@ceph.osd.6.smithi144.stdout:Sep 16 08:23:56 smithi144 systemd[1]: Stopping Ceph osd.6 for 5622b2fe-7402-11ef-bceb-c7b262605968... 2024-09-16T08:23:57.082 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:23:56 smithi144 ceph-mon[49149]: osd.6 marked itself down and dead 2024-09-16T08:23:57.082 INFO:journalctl@ceph.osd.6.smithi144.stdout:Sep 16 08:23:56 smithi144 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-6[35703]: 2024-09-16T08:23:56.737+0000 7fd9a847c700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.6 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-16T08:23:57.082 INFO:journalctl@ceph.osd.6.smithi144.stdout:Sep 16 08:23:56 smithi144 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-6[35703]: 2024-09-16T08:23:56.737+0000 7fd9a847c700 -1 osd.6 136 *** Got signal Terminated *** 2024-09-16T08:23:57.082 INFO:journalctl@ceph.osd.6.smithi144.stdout:Sep 16 08:23:56 smithi144 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-6[35703]: 2024-09-16T08:23:56.737+0000 7fd9a847c700 -1 osd.6 136 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-16T08:23:57.088 INFO:teuthology.orchestra.run.smithi028.stdout: 5 255845 9.41 MB/sec execute 535 sec latency 128.123 ms 2024-09-16T08:23:57.197 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:23:56 smithi028 ceph-mon[64115]: osd.6 marked itself down and dead 2024-09-16T08:23:57.966 INFO:journalctl@ceph.osd.6.smithi144.stdout:Sep 16 08:23:57 smithi144 podman[56628]: 2024-09-16 08:23:57.605257281 +0000 UTC m=+0.995573944 container died 7f9981be792fb2cf9583cc9baed8ff0ea8c20d7ee5ff0c176eb95ebb3d6035c0 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-6, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.openshift.tags=base centos centos-stream, build-date=2022-05-03T08:36:31.336870, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vendor=Red Hat, Inc., ceph=True, vcs-type=git, com.redhat.component=centos-stream-container, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, CEPH_POINT_RELEASE=-17.2.0, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, architecture=x86_64, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_CLEAN=True, io.buildah.version=1.19.8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, release=754, maintainer=Guillaume Abrioux , RELEASE=HEAD, version=8, distribution-scope=public, GIT_BRANCH=HEAD, GIT_REPO=https://github.com/ceph/ceph-container.git, name=centos-stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.expose-services=, io.k8s.display-name=CentOS Stream 8) 2024-09-16T08:23:58.089 INFO:teuthology.orchestra.run.smithi028.stdout: 5 255872 9.39 MB/sec execute 536 sec latency 1048.067 ms 2024-09-16T08:23:58.218 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:23:57 smithi028 ceph-mon[64115]: pgmap v252: 97 pgs: 97 active+clean; 343 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 5.4 KiB/s rd, 4.1 MiB/s wr, 399 op/s 2024-09-16T08:23:58.218 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:23:57 smithi028 ceph-mon[64115]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-16T08:23:58.218 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:23:57 smithi028 ceph-mon[64115]: osdmap e137: 8 total, 7 up, 8 in 2024-09-16T08:23:58.311 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:23:57 smithi144 ceph-mon[49149]: pgmap v252: 97 pgs: 97 active+clean; 343 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 5.4 KiB/s rd, 4.1 MiB/s wr, 399 op/s 2024-09-16T08:23:58.311 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:23:57 smithi144 ceph-mon[49149]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-16T08:23:58.311 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:23:57 smithi144 ceph-mon[49149]: osdmap e137: 8 total, 7 up, 8 in 2024-09-16T08:23:58.571 INFO:journalctl@ceph.osd.6.smithi144.stdout:Sep 16 08:23:58 smithi144 podman[56628]: 2024-09-16 08:23:58.31156693 +0000 UTC m=+1.701883581 container cleanup 7f9981be792fb2cf9583cc9baed8ff0ea8c20d7ee5ff0c176eb95ebb3d6035c0 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-6, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.k8s.display-name=CentOS Stream 8, ceph=True, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, version=8, CEPH_POINT_RELEASE=-17.2.0, vendor=Red Hat, Inc., vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, distribution-scope=public, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.expose-services=, GIT_CLEAN=True, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, architecture=x86_64, name=centos-stream, maintainer=Guillaume Abrioux , RELEASE=HEAD, vcs-type=git, io.buildah.version=1.19.8, release=754, io.openshift.tags=base centos centos-stream, com.redhat.component=centos-stream-container, GIT_BRANCH=HEAD, GIT_REPO=https://github.com/ceph/ceph-container.git, build-date=2022-05-03T08:36:31.336870) 2024-09-16T08:23:58.571 INFO:journalctl@ceph.osd.6.smithi144.stdout:Sep 16 08:23:58 smithi144 bash[56628]: ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-6 2024-09-16T08:23:58.831 INFO:journalctl@ceph.osd.6.smithi144.stdout:Sep 16 08:23:58 smithi144 podman[56661]: 2024-09-16 08:23:58.571687739 +0000 UTC m=+0.960666171 container remove 7f9981be792fb2cf9583cc9baed8ff0ea8c20d7ee5ff0c176eb95ebb3d6035c0 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-6, distribution-scope=public, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.component=centos-stream-container, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vcs-type=git, vendor=Red Hat, Inc., io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_CLEAN=True, RELEASE=HEAD, architecture=x86_64, io.k8s.display-name=CentOS Stream 8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.tags=base centos centos-stream, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=-17.2.0, GIT_BRANCH=HEAD, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, name=centos-stream, version=8, io.openshift.expose-services=, ceph=True, io.buildah.version=1.19.8, build-date=2022-05-03T08:36:31.336870, release=754) 2024-09-16T08:23:59.089 INFO:teuthology.orchestra.run.smithi028.stdout: 5 256243 9.40 MB/sec execute 537 sec latency 49.025 ms 2024-09-16T08:23:59.192 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:23:58 smithi144 ceph-mon[49149]: osdmap e138: 8 total, 7 up, 8 in 2024-09-16T08:23:59.192 INFO:journalctl@ceph.osd.6.smithi144.stdout:Sep 16 08:23:58 smithi144 podman[56750]: 2024-09-16 08:23:58.812909448 +0000 UTC m=+0.019019485 image pull 425e316ba8ebf74f04aa3640f7deb4fa3f293a78d0c8d097cb679eaf86ddca6c quay.ceph.io/ceph-ci/ceph@sha256:f7ad55f9ccd17ba66aadb19b4d291e67b60ea24e5d8a0c0e92941fd59c81e4ce 2024-09-16T08:23:59.192 INFO:journalctl@ceph.osd.6.smithi144.stdout:Sep 16 08:23:58 smithi144 podman[56750]: 2024-09-16 08:23:58.93120618 +0000 UTC m=+0.137316210 container create b058219d320512f39f63316966014b585403fa5b55bde25a203371837a1030d6 (image=quay.ceph.io/ceph-ci/ceph@sha256:f7ad55f9ccd17ba66aadb19b4d291e67b60ea24e5d8a0c0e92941fd59c81e4ce, name=ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-6-deactivate, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.schema-version=1.0, GIT_CLEAN=True, RELEASE=wip-guits-main-2024-09-13-1248-8293d73, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-16T08:23:59.192 INFO:journalctl@ceph.osd.6.smithi144.stdout:Sep 16 08:23:59 smithi144 podman[56750]: 2024-09-16 08:23:59.192647827 +0000 UTC m=+0.398757871 container init b058219d320512f39f63316966014b585403fa5b55bde25a203371837a1030d6 (image=quay.ceph.io/ceph-ci/ceph@sha256:f7ad55f9ccd17ba66aadb19b4d291e67b60ea24e5d8a0c0e92941fd59c81e4ce, name=ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-6-deactivate, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.build-date=20240731, RELEASE=wip-guits-main-2024-09-13-1248-8293d73, org.label-schema.vendor=CentOS, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0) 2024-09-16T08:23:59.228 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:23:58 smithi028 ceph-mon[64115]: osdmap e138: 8 total, 7 up, 8 in 2024-09-16T08:23:59.582 INFO:journalctl@ceph.osd.6.smithi144.stdout:Sep 16 08:23:59 smithi144 podman[56750]: 2024-09-16 08:23:59.196720696 +0000 UTC m=+0.402830729 container start b058219d320512f39f63316966014b585403fa5b55bde25a203371837a1030d6 (image=quay.ceph.io/ceph-ci/ceph@sha256:f7ad55f9ccd17ba66aadb19b4d291e67b60ea24e5d8a0c0e92941fd59c81e4ce, name=ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-6-deactivate, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, RELEASE=wip-guits-main-2024-09-13-1248-8293d73, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, ceph=True, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-16T08:23:59.582 INFO:journalctl@ceph.osd.6.smithi144.stdout:Sep 16 08:23:59 smithi144 podman[56750]: 2024-09-16 08:23:59.28423104 +0000 UTC m=+0.490341076 container attach b058219d320512f39f63316966014b585403fa5b55bde25a203371837a1030d6 (image=quay.ceph.io/ceph-ci/ceph@sha256:f7ad55f9ccd17ba66aadb19b4d291e67b60ea24e5d8a0c0e92941fd59c81e4ce, name=ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-6-deactivate, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, ceph=True, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-guits-main-2024-09-13-1248-8293d73, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, io.buildah.version=1.37.2) 2024-09-16T08:23:59.582 INFO:journalctl@ceph.osd.6.smithi144.stdout:Sep 16 08:23:59 smithi144 podman[56750]: 2024-09-16 08:23:59.405514938 +0000 UTC m=+0.611624981 container died b058219d320512f39f63316966014b585403fa5b55bde25a203371837a1030d6 (image=quay.ceph.io/ceph-ci/ceph@sha256:f7ad55f9ccd17ba66aadb19b4d291e67b60ea24e5d8a0c0e92941fd59c81e4ce, name=ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-6-deactivate, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_BRANCH=HEAD, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , RELEASE=wip-guits-main-2024-09-13-1248-8293d73, GIT_CLEAN=True, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-16T08:24:00.008 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:23:59 smithi028 ceph-mon[64115]: pgmap v255: 97 pgs: 5 active+undersized, 9 peering, 4 stale+active+clean, 6 active+undersized+degraded, 73 active+clean; 340 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 3.4 KiB/s rd, 3.4 MiB/s wr, 297 op/s; 39/1602 objects degraded (2.434%) 2024-09-16T08:24:00.009 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:23:59 smithi028 ceph-mon[64115]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-09-16T08:24:00.009 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:23:59 smithi028 ceph-mon[64115]: Health check failed: Degraded data redundancy: 39/1602 objects degraded (2.434%), 6 pgs degraded (PG_DEGRADED) 2024-09-16T08:24:00.089 INFO:teuthology.orchestra.run.smithi028.stdout: 5 256601 9.40 MB/sec execute 538 sec latency 43.532 ms 2024-09-16T08:24:00.191 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:23:59 smithi144 ceph-mon[49149]: pgmap v255: 97 pgs: 5 active+undersized, 9 peering, 4 stale+active+clean, 6 active+undersized+degraded, 73 active+clean; 340 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 3.4 KiB/s rd, 3.4 MiB/s wr, 297 op/s; 39/1602 objects degraded (2.434%) 2024-09-16T08:24:00.192 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:23:59 smithi144 ceph-mon[49149]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-09-16T08:24:00.192 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:23:59 smithi144 ceph-mon[49149]: Health check failed: Degraded data redundancy: 39/1602 objects degraded (2.434%), 6 pgs degraded (PG_DEGRADED) 2024-09-16T08:24:00.192 INFO:journalctl@ceph.osd.6.smithi144.stdout:Sep 16 08:24:00 smithi144 podman[56773]: 2024-09-16 08:24:00.156543648 +0000 UTC m=+0.745074846 container remove b058219d320512f39f63316966014b585403fa5b55bde25a203371837a1030d6 (image=quay.ceph.io/ceph-ci/ceph@sha256:f7ad55f9ccd17ba66aadb19b4d291e67b60ea24e5d8a0c0e92941fd59c81e4ce, name=ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-6-deactivate, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, ceph=True, org.label-schema.schema-version=1.0, RELEASE=wip-guits-main-2024-09-13-1248-8293d73, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-16T08:24:00.192 INFO:journalctl@ceph.osd.6.smithi144.stdout:Sep 16 08:24:00 smithi144 systemd[1]: ceph-5622b2fe-7402-11ef-bceb-c7b262605968@osd.6.service: Deactivated successfully. 2024-09-16T08:24:00.192 INFO:journalctl@ceph.osd.6.smithi144.stdout:Sep 16 08:24:00 smithi144 systemd[1]: Stopped Ceph osd.6 for 5622b2fe-7402-11ef-bceb-c7b262605968. 2024-09-16T08:24:00.192 INFO:journalctl@ceph.osd.6.smithi144.stdout:Sep 16 08:24:00 smithi144 systemd[1]: ceph-5622b2fe-7402-11ef-bceb-c7b262605968@osd.6.service: Consumed 4min 32.642s CPU time. 2024-09-16T08:24:00.582 INFO:journalctl@ceph.osd.6.smithi144.stdout:Sep 16 08:24:00 smithi144 systemd[1]: Starting Ceph osd.6 for 5622b2fe-7402-11ef-bceb-c7b262605968... 2024-09-16T08:24:00.935 INFO:journalctl@ceph.osd.6.smithi144.stdout:Sep 16 08:24:00 smithi144 podman[56869]: 2024-09-16 08:24:00.514319098 +0000 UTC m=+0.017701481 image pull 425e316ba8ebf74f04aa3640f7deb4fa3f293a78d0c8d097cb679eaf86ddca6c quay.ceph.io/ceph-ci/ceph@sha256:f7ad55f9ccd17ba66aadb19b4d291e67b60ea24e5d8a0c0e92941fd59c81e4ce 2024-09-16T08:24:00.936 INFO:journalctl@ceph.osd.6.smithi144.stdout:Sep 16 08:24:00 smithi144 podman[56869]: 2024-09-16 08:24:00.641488301 +0000 UTC m=+0.144870680 container create 02b1da390962bba565e558559989b9f66cf3ef4d2b10ef69d647693dbb45ac58 (image=quay.ceph.io/ceph-ci/ceph@sha256:f7ad55f9ccd17ba66aadb19b4d291e67b60ea24e5d8a0c0e92941fd59c81e4ce, name=ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-6-activate, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, ceph=True, RELEASE=wip-guits-main-2024-09-13-1248-8293d73, org.label-schema.schema-version=1.0) 2024-09-16T08:24:01.089 INFO:teuthology.orchestra.run.smithi028.stdout: 5 256976 9.40 MB/sec execute 539 sec latency 47.101 ms 2024-09-16T08:24:01.332 INFO:journalctl@ceph.osd.6.smithi144.stdout:Sep 16 08:24:00 smithi144 podman[56869]: 2024-09-16 08:24:00.936099756 +0000 UTC m=+0.439482153 container init 02b1da390962bba565e558559989b9f66cf3ef4d2b10ef69d647693dbb45ac58 (image=quay.ceph.io/ceph-ci/ceph@sha256:f7ad55f9ccd17ba66aadb19b4d291e67b60ea24e5d8a0c0e92941fd59c81e4ce, name=ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-6-activate, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, ceph=True, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_CLEAN=True, RELEASE=wip-guits-main-2024-09-13-1248-8293d73, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-16T08:24:01.332 INFO:journalctl@ceph.osd.6.smithi144.stdout:Sep 16 08:24:00 smithi144 podman[56869]: 2024-09-16 08:24:00.942351356 +0000 UTC m=+0.445733745 container start 02b1da390962bba565e558559989b9f66cf3ef4d2b10ef69d647693dbb45ac58 (image=quay.ceph.io/ceph-ci/ceph@sha256:f7ad55f9ccd17ba66aadb19b4d291e67b60ea24e5d8a0c0e92941fd59c81e4ce, name=ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-6-activate, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, RELEASE=wip-guits-main-2024-09-13-1248-8293d73, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, ceph=True, GIT_CLEAN=True, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=) 2024-09-16T08:24:01.332 INFO:journalctl@ceph.osd.6.smithi144.stdout:Sep 16 08:24:01 smithi144 podman[56869]: 2024-09-16 08:24:01.044657608 +0000 UTC m=+0.548039988 container attach 02b1da390962bba565e558559989b9f66cf3ef4d2b10ef69d647693dbb45ac58 (image=quay.ceph.io/ceph-ci/ceph@sha256:f7ad55f9ccd17ba66aadb19b4d291e67b60ea24e5d8a0c0e92941fd59c81e4ce, name=ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-6-activate, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=wip-guits-main-2024-09-13-1248-8293d73, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, ceph=True, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-16T08:24:01.332 INFO:journalctl@ceph.osd.6.smithi144.stdout:Sep 16 08:24:01 smithi144 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-6-activate[56884]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-16T08:24:01.332 INFO:journalctl@ceph.osd.6.smithi144.stdout:Sep 16 08:24:01 smithi144 bash[56869]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-16T08:24:01.332 INFO:journalctl@ceph.osd.6.smithi144.stdout:Sep 16 08:24:01 smithi144 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-6-activate[56884]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-16T08:24:01.332 INFO:journalctl@ceph.osd.6.smithi144.stdout:Sep 16 08:24:01 smithi144 bash[56869]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-16T08:24:02.082 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:24:01 smithi144 ceph-mon[49149]: pgmap v256: 97 pgs: 7 active+undersized, 9 peering, 2 stale+active+clean, 12 active+undersized+degraded, 67 active+clean; 344 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 3.6 KiB/s rd, 4.0 MiB/s wr, 455 op/s; 89/1575 objects degraded (5.651%) 2024-09-16T08:24:02.089 INFO:teuthology.orchestra.run.smithi028.stdout: 5 257326 9.40 MB/sec execute 540 sec latency 75.513 ms 2024-09-16T08:24:02.097 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:24:01 smithi028 ceph-mon[64115]: pgmap v256: 97 pgs: 7 active+undersized, 9 peering, 2 stale+active+clean, 12 active+undersized+degraded, 67 active+clean; 344 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 3.6 KiB/s rd, 4.0 MiB/s wr, 455 op/s; 89/1575 objects degraded (5.651%) 2024-09-16T08:24:03.089 INFO:teuthology.orchestra.run.smithi028.stdout: 5 257559 9.40 MB/sec execute 541 sec latency 64.738 ms 2024-09-16T08:24:03.427 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:24:03 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:24:03.427 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:24:03 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-16T08:24:03.427 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:24:03 smithi028 ceph-mon[64115]: pgmap v257: 97 pgs: 11 active+undersized, 5 peering, 19 active+undersized+degraded, 62 active+clean; 343 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 5.2 KiB/s rd, 3.0 MiB/s wr, 316 op/s; 160/1599 objects degraded (10.006%) 2024-09-16T08:24:03.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:24:03 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:24:03.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:24:03 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-16T08:24:03.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:24:03 smithi144 ceph-mon[49149]: pgmap v257: 97 pgs: 11 active+undersized, 5 peering, 19 active+undersized+degraded, 62 active+clean; 343 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 5.2 KiB/s rd, 3.0 MiB/s wr, 316 op/s; 160/1599 objects degraded (10.006%) 2024-09-16T08:24:03.582 INFO:journalctl@ceph.osd.6.smithi144.stdout:Sep 16 08:24:03 smithi144 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-6-activate[56884]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-6 2024-09-16T08:24:03.582 INFO:journalctl@ceph.osd.6.smithi144.stdout:Sep 16 08:24:03 smithi144 bash[56869]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-6 2024-09-16T08:24:03.582 INFO:journalctl@ceph.osd.6.smithi144.stdout:Sep 16 08:24:03 smithi144 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-6-activate[56884]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-6 --no-mon-config --dev /dev/mapper/vg_nvme-lv_2 2024-09-16T08:24:03.582 INFO:journalctl@ceph.osd.6.smithi144.stdout:Sep 16 08:24:03 smithi144 bash[56869]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-6 --no-mon-config --dev /dev/mapper/vg_nvme-lv_2 2024-09-16T08:24:04.082 INFO:journalctl@ceph.osd.6.smithi144.stdout:Sep 16 08:24:03 smithi144 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-6-activate[56884]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_2 2024-09-16T08:24:04.082 INFO:journalctl@ceph.osd.6.smithi144.stdout:Sep 16 08:24:03 smithi144 bash[56869]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_2 2024-09-16T08:24:04.082 INFO:journalctl@ceph.osd.6.smithi144.stdout:Sep 16 08:24:03 smithi144 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-6-activate[56884]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-09-16T08:24:04.082 INFO:journalctl@ceph.osd.6.smithi144.stdout:Sep 16 08:24:03 smithi144 bash[56869]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-09-16T08:24:04.082 INFO:journalctl@ceph.osd.6.smithi144.stdout:Sep 16 08:24:03 smithi144 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-6-activate[56884]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-6/block 2024-09-16T08:24:04.082 INFO:journalctl@ceph.osd.6.smithi144.stdout:Sep 16 08:24:03 smithi144 bash[56869]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-6/block 2024-09-16T08:24:04.082 INFO:journalctl@ceph.osd.6.smithi144.stdout:Sep 16 08:24:03 smithi144 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-6-activate[56884]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-6 2024-09-16T08:24:04.082 INFO:journalctl@ceph.osd.6.smithi144.stdout:Sep 16 08:24:03 smithi144 bash[56869]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-6 2024-09-16T08:24:04.082 INFO:journalctl@ceph.osd.6.smithi144.stdout:Sep 16 08:24:03 smithi144 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-6-activate[56884]: --> ceph-volume raw activate successful for osd ID: 6 2024-09-16T08:24:04.082 INFO:journalctl@ceph.osd.6.smithi144.stdout:Sep 16 08:24:03 smithi144 bash[56869]: --> ceph-volume raw activate successful for osd ID: 6 2024-09-16T08:24:04.083 INFO:journalctl@ceph.osd.6.smithi144.stdout:Sep 16 08:24:03 smithi144 podman[56869]: 2024-09-16 08:24:03.792889769 +0000 UTC m=+3.296272148 container died 02b1da390962bba565e558559989b9f66cf3ef4d2b10ef69d647693dbb45ac58 (image=quay.ceph.io/ceph-ci/ceph@sha256:f7ad55f9ccd17ba66aadb19b4d291e67b60ea24e5d8a0c0e92941fd59c81e4ce, name=ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-6-activate, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, ceph=True, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=wip-guits-main-2024-09-13-1248-8293d73, GIT_BRANCH=HEAD) 2024-09-16T08:24:04.089 INFO:teuthology.orchestra.run.smithi028.stdout: 5 257882 9.41 MB/sec execute 542 sec latency 67.233 ms 2024-09-16T08:24:04.446 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:24:04 smithi028 ceph-mon[64115]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-09-16T08:24:04.568 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:24:04 smithi144 ceph-mon[49149]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-09-16T08:24:04.832 INFO:journalctl@ceph.osd.6.smithi144.stdout:Sep 16 08:24:04 smithi144 podman[57035]: 2024-09-16 08:24:04.568671899 +0000 UTC m=+0.769246803 container remove 02b1da390962bba565e558559989b9f66cf3ef4d2b10ef69d647693dbb45ac58 (image=quay.ceph.io/ceph-ci/ceph@sha256:f7ad55f9ccd17ba66aadb19b4d291e67b60ea24e5d8a0c0e92941fd59c81e4ce, name=ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-6-activate, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, CEPH_POINT_RELEASE=, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, io.buildah.version=1.37.2, RELEASE=wip-guits-main-2024-09-13-1248-8293d73, GIT_CLEAN=True, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0) 2024-09-16T08:24:04.832 INFO:journalctl@ceph.osd.6.smithi144.stdout:Sep 16 08:24:04 smithi144 podman[57091]: 2024-09-16 08:24:04.688833395 +0000 UTC m=+0.016926444 image pull 425e316ba8ebf74f04aa3640f7deb4fa3f293a78d0c8d097cb679eaf86ddca6c quay.ceph.io/ceph-ci/ceph@sha256:f7ad55f9ccd17ba66aadb19b4d291e67b60ea24e5d8a0c0e92941fd59c81e4ce 2024-09-16T08:24:04.832 INFO:journalctl@ceph.osd.6.smithi144.stdout:Sep 16 08:24:04 smithi144 podman[57091]: 2024-09-16 08:24:04.811513091 +0000 UTC m=+0.139606131 container create b301bf0731f04bbacf337c0c418bc53a126ea72faaf3caf62c415f973fa701e1 (image=quay.ceph.io/ceph-ci/ceph@sha256:f7ad55f9ccd17ba66aadb19b4d291e67b60ea24e5d8a0c0e92941fd59c81e4ce, name=ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-6, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, ceph=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , RELEASE=wip-guits-main-2024-09-13-1248-8293d73, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, GIT_CLEAN=True, org.label-schema.license=GPLv2) 2024-09-16T08:24:05.085 INFO:teuthology.orchestra.run.smithi028.stdout:true 2024-09-16T08:24:05.089 INFO:teuthology.orchestra.run.smithi028.stdout: 5 258252 9.40 MB/sec execute 543 sec latency 56.499 ms 2024-09-16T08:24:05.512 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:24:05 smithi028 ceph-mon[64115]: pgmap v258: 97 pgs: 11 active+undersized, 24 active+undersized+degraded, 62 active+clean; 344 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 5.2 KiB/s rd, 4.2 MiB/s wr, 412 op/s; 211/1665 objects degraded (12.673%) 2024-09-16T08:24:05.512 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:24:05 smithi028 ceph-mon[64115]: Health check update: Degraded data redundancy: 160/1599 objects degraded (10.006%), 19 pgs degraded (PG_DEGRADED) 2024-09-16T08:24:05.512 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:24:05 smithi028 ceph-mon[64115]: from='client.34404 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T08:24:05.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:24:05 smithi144 ceph-mon[49149]: pgmap v258: 97 pgs: 11 active+undersized, 24 active+undersized+degraded, 62 active+clean; 344 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 5.2 KiB/s rd, 4.2 MiB/s wr, 412 op/s; 211/1665 objects degraded (12.673%) 2024-09-16T08:24:05.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:24:05 smithi144 ceph-mon[49149]: Health check update: Degraded data redundancy: 160/1599 objects degraded (10.006%), 19 pgs degraded (PG_DEGRADED) 2024-09-16T08:24:05.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:24:05 smithi144 ceph-mon[49149]: from='client.34404 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T08:24:05.582 INFO:journalctl@ceph.osd.6.smithi144.stdout:Sep 16 08:24:05 smithi144 podman[57091]: 2024-09-16 08:24:05.347605911 +0000 UTC m=+0.675698954 container init b301bf0731f04bbacf337c0c418bc53a126ea72faaf3caf62c415f973fa701e1 (image=quay.ceph.io/ceph-ci/ceph@sha256:f7ad55f9ccd17ba66aadb19b4d291e67b60ea24e5d8a0c0e92941fd59c81e4ce, name=ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-6, ceph=True, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=wip-guits-main-2024-09-13-1248-8293d73, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=) 2024-09-16T08:24:05.582 INFO:journalctl@ceph.osd.6.smithi144.stdout:Sep 16 08:24:05 smithi144 podman[57091]: 2024-09-16 08:24:05.351690706 +0000 UTC m=+0.679783748 container start b301bf0731f04bbacf337c0c418bc53a126ea72faaf3caf62c415f973fa701e1 (image=quay.ceph.io/ceph-ci/ceph@sha256:f7ad55f9ccd17ba66aadb19b4d291e67b60ea24e5d8a0c0e92941fd59c81e4ce, name=ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-6, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, ceph=True, RELEASE=wip-guits-main-2024-09-13-1248-8293d73, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731) 2024-09-16T08:24:05.582 INFO:journalctl@ceph.osd.6.smithi144.stdout:Sep 16 08:24:05 smithi144 ceph-osd[57109]: -- 172.21.15.144:0/1580079175 <== mon.1 v2:172.21.15.144:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x555b934e01e0 con 0x555b934d2000 2024-09-16T08:24:05.582 INFO:journalctl@ceph.osd.6.smithi144.stdout:Sep 16 08:24:05 smithi144 bash[57091]: b301bf0731f04bbacf337c0c418bc53a126ea72faaf3caf62c415f973fa701e1 2024-09-16T08:24:05.583 INFO:journalctl@ceph.osd.6.smithi144.stdout:Sep 16 08:24:05 smithi144 systemd[1]: Started Ceph osd.6 for 5622b2fe-7402-11ef-bceb-c7b262605968. 2024-09-16T08:24:06.050 INFO:teuthology.orchestra.run.smithi028.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-16T08:24:06.050 INFO:teuthology.orchestra.run.smithi028.stdout:alertmanager.smithi028 smithi028 *:9093,9094 running (8m) 114s ago 17m 18.6M - 0.27.0 11f11916f8cd a14fcb4cb2b6 2024-09-16T08:24:06.050 INFO:teuthology.orchestra.run.smithi028.stdout:crash.smithi028 smithi028 running (5m) 114s ago 17m 6899k - 19.3.0-4917-g8293d73f 425e316ba8eb 4bb89df2a867 2024-09-16T08:24:06.050 INFO:teuthology.orchestra.run.smithi028.stdout:crash.smithi144 smithi144 running (5m) 35s ago 16m 6895k - 19.3.0-4917-g8293d73f 425e316ba8eb 24a96875d7af 2024-09-16T08:24:06.050 INFO:teuthology.orchestra.run.smithi028.stdout:grafana.smithi028 smithi028 *:3000 running (8m) 114s ago 17m 45.9M - dad864ee21e9 b55ba8b944a1 2024-09-16T08:24:06.050 INFO:teuthology.orchestra.run.smithi028.stdout:mds.foofs.smithi028.srtgev smithi028 running (11m) 114s ago 11m 38.5M - 17.2.0 e1d6a67b021e 250e7a7f754c 2024-09-16T08:24:06.050 INFO:teuthology.orchestra.run.smithi028.stdout:mds.foofs.smithi144.oozgsi smithi144 running (11m) 35s ago 11m 17.5M - 17.2.0 e1d6a67b021e ecf4c2bb0fdc 2024-09-16T08:24:06.050 INFO:teuthology.orchestra.run.smithi028.stdout:mgr.smithi028.azvyzy smithi028 *:8443,9283,8765 running (7m) 114s ago 18m 519M - 19.3.0-4917-g8293d73f 425e316ba8eb e01d4382d03f 2024-09-16T08:24:06.050 INFO:teuthology.orchestra.run.smithi028.stdout:mgr.smithi144.utzszj smithi144 *:8443,9283,8765 running (6m) 35s ago 16m 457M - 19.3.0-4917-g8293d73f 425e316ba8eb 71f7f802bffb 2024-09-16T08:24:06.050 INFO:teuthology.orchestra.run.smithi028.stdout:mon.smithi028 smithi028 running (6m) 114s ago 18m 49.8M 2048M 19.3.0-4917-g8293d73f 425e316ba8eb 14efcedc0088 2024-09-16T08:24:06.051 INFO:teuthology.orchestra.run.smithi028.stdout:mon.smithi144 smithi144 running (6m) 35s ago 16m 46.5M 2048M 19.3.0-4917-g8293d73f 425e316ba8eb c081c5ed44b5 2024-09-16T08:24:06.051 INFO:teuthology.orchestra.run.smithi028.stdout:nfs.foo.0.0.smithi028.dkttgj smithi028 *:2049 running (11m) 114s ago 11m 285M - 4.0 e1d6a67b021e b5dccc45a6ed 2024-09-16T08:24:06.051 INFO:teuthology.orchestra.run.smithi028.stdout:nfs.foo.1.0.smithi144.epipkg smithi144 *:2049 running (11m) 35s ago 11m 87.4M - 4.0 e1d6a67b021e faa89e61f50e 2024-09-16T08:24:06.051 INFO:teuthology.orchestra.run.smithi028.stdout:node-exporter.smithi028 smithi028 *:9100 running (8m) 114s ago 17m 9.92M - 1.7.0 72c9c2088986 9e2b8e8a63a9 2024-09-16T08:24:06.051 INFO:teuthology.orchestra.run.smithi028.stdout:node-exporter.smithi144 smithi144 *:9100 running (7m) 35s ago 16m 10.8M - 1.7.0 72c9c2088986 78834c5e992f 2024-09-16T08:24:06.051 INFO:teuthology.orchestra.run.smithi028.stdout:osd.0 smithi028 running (4m) 114s ago 16m 354M 1937M 19.3.0-4917-g8293d73f 425e316ba8eb 2fc02ba4f14f 2024-09-16T08:24:06.051 INFO:teuthology.orchestra.run.smithi028.stdout:osd.1 smithi028 running (3m) 114s ago 15m 266M 1937M 19.3.0-4917-g8293d73f 425e316ba8eb df34bd603d8f 2024-09-16T08:24:06.051 INFO:teuthology.orchestra.run.smithi028.stdout:osd.2 smithi028 running (2m) 114s ago 15m 220M 1937M 19.3.0-4917-g8293d73f 425e316ba8eb c92be6327b12 2024-09-16T08:24:06.051 INFO:teuthology.orchestra.run.smithi028.stdout:osd.3 smithi028 running (2m) 114s ago 14m 11.2M 1937M 19.3.0-4917-g8293d73f 425e316ba8eb bd84ba1224f2 2024-09-16T08:24:06.051 INFO:teuthology.orchestra.run.smithi028.stdout:osd.4 smithi144 running (78s) 35s ago 14m 270M 2704M 19.3.0-4917-g8293d73f 425e316ba8eb c48cb68e901e 2024-09-16T08:24:06.051 INFO:teuthology.orchestra.run.smithi028.stdout:osd.5 smithi144 running (39s) 35s ago 13m 12.1M 2704M 19.3.0-4917-g8293d73f 425e316ba8eb 59ec97fb3cf3 2024-09-16T08:24:06.051 INFO:teuthology.orchestra.run.smithi028.stdout:osd.6 smithi144 starting - - - 2704M 2024-09-16T08:24:06.051 INFO:teuthology.orchestra.run.smithi028.stdout:osd.7 smithi144 running (13m) 35s ago 13m 784M 2704M 17.2.0 e1d6a67b021e dada74ddbe4c 2024-09-16T08:24:06.051 INFO:teuthology.orchestra.run.smithi028.stdout:prometheus.smithi028 smithi028 *:9095 running (6m) 114s ago 16m 55.3M - 2.51.0 1d3b7f56885b 42667809df2b 2024-09-16T08:24:06.089 INFO:teuthology.orchestra.run.smithi028.stdout: 5 258495 9.39 MB/sec execute 544 sec latency 56.790 ms 2024-09-16T08:24:06.526 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:24:06 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:24:06.526 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:24:06 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:24:06.526 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:24:06 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-16T08:24:06.526 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:24:06 smithi028 ceph-mon[64115]: from='client.44347 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T08:24:06.526 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:24:06 smithi028 ceph-mon[64115]: from='client.34412 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T08:24:06.636 INFO:teuthology.orchestra.run.smithi028.stdout:{ 2024-09-16T08:24:06.636 INFO:teuthology.orchestra.run.smithi028.stdout: "mon": { 2024-09-16T08:24:06.636 INFO:teuthology.orchestra.run.smithi028.stdout: "ceph version 19.3.0-4917-g8293d73f (8293d73f8690540e843a81caec373f9cc29cf705) squid (dev)": 2 2024-09-16T08:24:06.636 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-09-16T08:24:06.636 INFO:teuthology.orchestra.run.smithi028.stdout: "mgr": { 2024-09-16T08:24:06.636 INFO:teuthology.orchestra.run.smithi028.stdout: "ceph version 19.3.0-4917-g8293d73f (8293d73f8690540e843a81caec373f9cc29cf705) squid (dev)": 2 2024-09-16T08:24:06.636 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-09-16T08:24:06.637 INFO:teuthology.orchestra.run.smithi028.stdout: "osd": { 2024-09-16T08:24:06.637 INFO:teuthology.orchestra.run.smithi028.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 1, 2024-09-16T08:24:06.637 INFO:teuthology.orchestra.run.smithi028.stdout: "ceph version 19.3.0-4917-g8293d73f (8293d73f8690540e843a81caec373f9cc29cf705) squid (dev)": 6 2024-09-16T08:24:06.637 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-09-16T08:24:06.637 INFO:teuthology.orchestra.run.smithi028.stdout: "mds": { 2024-09-16T08:24:06.637 INFO:teuthology.orchestra.run.smithi028.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-16T08:24:06.637 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-09-16T08:24:06.637 INFO:teuthology.orchestra.run.smithi028.stdout: "overall": { 2024-09-16T08:24:06.637 INFO:teuthology.orchestra.run.smithi028.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3, 2024-09-16T08:24:06.637 INFO:teuthology.orchestra.run.smithi028.stdout: "ceph version 19.3.0-4917-g8293d73f (8293d73f8690540e843a81caec373f9cc29cf705) squid (dev)": 10 2024-09-16T08:24:06.637 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-09-16T08:24:06.637 INFO:teuthology.orchestra.run.smithi028.stdout:} 2024-09-16T08:24:06.832 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:24:06 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:24:06.832 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:24:06 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:24:06.832 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:24:06 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-16T08:24:06.832 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:24:06 smithi144 ceph-mon[49149]: from='client.44347 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T08:24:06.832 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:24:06 smithi144 ceph-mon[49149]: from='client.34412 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T08:24:07.089 INFO:teuthology.orchestra.run.smithi028.stdout: 5 258722 9.38 MB/sec execute 545 sec latency 45.130 ms 2024-09-16T08:24:07.173 INFO:teuthology.orchestra.run.smithi028.stdout:{ 2024-09-16T08:24:07.174 INFO:teuthology.orchestra.run.smithi028.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:f7ad55f9ccd17ba66aadb19b4d291e67b60ea24e5d8a0c0e92941fd59c81e4ce", 2024-09-16T08:24:07.174 INFO:teuthology.orchestra.run.smithi028.stdout: "in_progress": true, 2024-09-16T08:24:07.174 INFO:teuthology.orchestra.run.smithi028.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-16T08:24:07.174 INFO:teuthology.orchestra.run.smithi028.stdout: "services_complete": [ 2024-09-16T08:24:07.174 INFO:teuthology.orchestra.run.smithi028.stdout: "crash", 2024-09-16T08:24:07.174 INFO:teuthology.orchestra.run.smithi028.stdout: "mon", 2024-09-16T08:24:07.174 INFO:teuthology.orchestra.run.smithi028.stdout: "mgr" 2024-09-16T08:24:07.174 INFO:teuthology.orchestra.run.smithi028.stdout: ], 2024-09-16T08:24:07.174 INFO:teuthology.orchestra.run.smithi028.stdout: "progress": "12/23 daemons upgraded", 2024-09-16T08:24:07.174 INFO:teuthology.orchestra.run.smithi028.stdout: "message": "Currently upgrading osd daemons", 2024-09-16T08:24:07.174 INFO:teuthology.orchestra.run.smithi028.stdout: "is_paused": false 2024-09-16T08:24:07.175 INFO:teuthology.orchestra.run.smithi028.stdout:} 2024-09-16T08:24:07.332 INFO:journalctl@ceph.osd.6.smithi144.stdout:Sep 16 08:24:06 smithi144 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-6[57105]: 2024-09-16T08:24:06.989+0000 7fc05d2c7640 -1 Falling back to public interface 2024-09-16T08:24:07.786 INFO:teuthology.orchestra.run.smithi028.stdout:HEALTH_WARN 1 osds down; Degraded data redundancy: 208/1668 objects degraded (12.470%), 24 pgs degraded 2024-09-16T08:24:07.786 INFO:teuthology.orchestra.run.smithi028.stdout:[WRN] OSD_DOWN: 1 osds down 2024-09-16T08:24:07.786 INFO:teuthology.orchestra.run.smithi028.stdout: osd.6 (root=default,host=smithi144) is down 2024-09-16T08:24:07.786 INFO:teuthology.orchestra.run.smithi028.stdout:[WRN] PG_DEGRADED: Degraded data redundancy: 208/1668 objects degraded (12.470%), 24 pgs degraded 2024-09-16T08:24:07.786 INFO:teuthology.orchestra.run.smithi028.stdout: pg 1.0 is active+undersized+degraded, acting [7,0] 2024-09-16T08:24:07.786 INFO:teuthology.orchestra.run.smithi028.stdout: pg 2.6 is active+undersized+degraded, acting [1,4] 2024-09-16T08:24:07.786 INFO:teuthology.orchestra.run.smithi028.stdout: pg 2.7 is active+undersized+degraded, acting [7,2] 2024-09-16T08:24:07.786 INFO:teuthology.orchestra.run.smithi028.stdout: pg 2.11 is active+undersized+degraded, acting [4,1] 2024-09-16T08:24:07.787 INFO:teuthology.orchestra.run.smithi028.stdout: pg 2.14 is active+undersized+degraded, acting [3,5] 2024-09-16T08:24:07.787 INFO:teuthology.orchestra.run.smithi028.stdout: pg 2.16 is active+undersized+degraded, acting [5,2] 2024-09-16T08:24:07.787 INFO:teuthology.orchestra.run.smithi028.stdout: pg 2.17 is active+undersized+degraded, acting [5,2] 2024-09-16T08:24:07.787 INFO:teuthology.orchestra.run.smithi028.stdout: pg 2.19 is active+undersized+degraded, acting [3,0] 2024-09-16T08:24:07.787 INFO:teuthology.orchestra.run.smithi028.stdout: pg 2.1a is active+undersized+degraded, acting [4,7] 2024-09-16T08:24:07.787 INFO:teuthology.orchestra.run.smithi028.stdout: pg 2.1b is active+undersized+degraded, acting [3,2] 2024-09-16T08:24:07.787 INFO:teuthology.orchestra.run.smithi028.stdout: pg 2.1d is active+undersized+degraded, acting [7,0] 2024-09-16T08:24:07.787 INFO:teuthology.orchestra.run.smithi028.stdout: pg 3.0 is active+undersized+degraded, acting [1,2] 2024-09-16T08:24:07.787 INFO:teuthology.orchestra.run.smithi028.stdout: pg 3.2 is active+undersized+degraded, acting [3,5] 2024-09-16T08:24:07.787 INFO:teuthology.orchestra.run.smithi028.stdout: pg 3.3 is active+undersized+degraded, acting [4,0] 2024-09-16T08:24:07.787 INFO:teuthology.orchestra.run.smithi028.stdout: pg 3.a is active+undersized+degraded, acting [4,1] 2024-09-16T08:24:07.787 INFO:teuthology.orchestra.run.smithi028.stdout: pg 3.c is active+undersized+degraded, acting [5,3] 2024-09-16T08:24:07.787 INFO:teuthology.orchestra.run.smithi028.stdout: pg 3.d is active+undersized+degraded, acting [7,5] 2024-09-16T08:24:07.788 INFO:teuthology.orchestra.run.smithi028.stdout: pg 3.10 is active+undersized+degraded, acting [0,5] 2024-09-16T08:24:07.788 INFO:teuthology.orchestra.run.smithi028.stdout: pg 3.11 is active+undersized+degraded, acting [7,4] 2024-09-16T08:24:07.788 INFO:teuthology.orchestra.run.smithi028.stdout: pg 3.14 is active+undersized+degraded, acting [4,7] 2024-09-16T08:24:07.788 INFO:teuthology.orchestra.run.smithi028.stdout: pg 3.1d is active+undersized+degraded, acting [5,4] 2024-09-16T08:24:07.788 INFO:teuthology.orchestra.run.smithi028.stdout: pg 3.1e is active+undersized+degraded, acting [3,2] 2024-09-16T08:24:07.788 INFO:teuthology.orchestra.run.smithi028.stdout: pg 4.5 is active+undersized+degraded, acting [4,7] 2024-09-16T08:24:07.788 INFO:teuthology.orchestra.run.smithi028.stdout: pg 4.a is active+undersized+degraded, acting [1,7] 2024-09-16T08:24:07.805 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:24:07 smithi028 ceph-mon[64115]: from='client.? 172.21.15.28:0/3254715656' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-16T08:24:07.806 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:24:07 smithi028 ceph-mon[64115]: pgmap v259: 97 pgs: 11 active+undersized, 24 active+undersized+degraded, 62 active+clean; 347 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 7.1 KiB/s rd, 4.3 MiB/s wr, 461 op/s; 208/1668 objects degraded (12.470%) 2024-09-16T08:24:07.806 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:24:07 smithi028 ceph-mon[64115]: from='client.34420 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T08:24:07.832 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:24:07 smithi144 ceph-mon[49149]: from='client.? 172.21.15.28:0/3254715656' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-16T08:24:07.832 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:24:07 smithi144 ceph-mon[49149]: pgmap v259: 97 pgs: 11 active+undersized, 24 active+undersized+degraded, 62 active+clean; 347 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 7.1 KiB/s rd, 4.3 MiB/s wr, 461 op/s; 208/1668 objects degraded (12.470%) 2024-09-16T08:24:07.832 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:24:07 smithi144 ceph-mon[49149]: from='client.34420 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T08:24:08.089 INFO:teuthology.orchestra.run.smithi028.stdout: 5 259017 9.38 MB/sec execute 546 sec latency 58.785 ms 2024-09-16T08:24:08.569 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:24:08 smithi028 ceph-mon[64115]: from='client.? 172.21.15.28:0/2248018248' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-16T08:24:08.831 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:24:08 smithi144 ceph-mon[49149]: from='client.? 172.21.15.28:0/2248018248' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-16T08:24:09.089 INFO:teuthology.orchestra.run.smithi028.stdout: 5 259317 9.38 MB/sec execute 547 sec latency 63.766 ms 2024-09-16T08:24:09.591 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:24:09 smithi028 ceph-mon[64115]: pgmap v260: 97 pgs: 11 active+undersized, 24 active+undersized+degraded, 62 active+clean; 346 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 6.5 KiB/s rd, 4.7 MiB/s wr, 480 op/s; 204/1653 objects degraded (12.341%) 2024-09-16T08:24:09.832 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:24:09 smithi144 ceph-mon[49149]: pgmap v260: 97 pgs: 11 active+undersized, 24 active+undersized+degraded, 62 active+clean; 346 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 6.5 KiB/s rd, 4.7 MiB/s wr, 480 op/s; 204/1653 objects degraded (12.341%) 2024-09-16T08:24:10.089 INFO:teuthology.orchestra.run.smithi028.stdout: 5 259668 9.38 MB/sec execute 548 sec latency 62.140 ms 2024-09-16T08:24:10.883 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:24:10 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:24:10.883 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:24:10 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:24:10.883 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:24:10 smithi028 ceph-mon[64115]: Health check update: Degraded data redundancy: 204/1653 objects degraded (12.341%), 24 pgs degraded (PG_DEGRADED) 2024-09-16T08:24:10.883 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:24:10 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:24:10.883 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:24:10 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:24:11.089 INFO:teuthology.orchestra.run.smithi028.stdout: 5 259912 9.37 MB/sec execute 549 sec latency 65.873 ms 2024-09-16T08:24:11.129 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:24:10 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:24:11.129 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:24:10 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:24:11.129 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:24:10 smithi144 ceph-mon[49149]: Health check update: Degraded data redundancy: 204/1653 objects degraded (12.341%), 24 pgs degraded (PG_DEGRADED) 2024-09-16T08:24:11.130 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:24:10 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:24:11.130 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:24:10 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:24:11.892 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:24:11 smithi028 ceph-mon[64115]: pgmap v261: 97 pgs: 11 active+undersized, 24 active+undersized+degraded, 62 active+clean; 340 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.5 MiB/s wr, 457 op/s; 198/1635 objects degraded (12.110%) 2024-09-16T08:24:12.089 INFO:teuthology.orchestra.run.smithi028.stdout: 5 260222 9.37 MB/sec execute 550 sec latency 103.262 ms 2024-09-16T08:24:12.130 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:24:11 smithi144 ceph-mon[49149]: pgmap v261: 97 pgs: 11 active+undersized, 24 active+undersized+degraded, 62 active+clean; 340 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.5 MiB/s wr, 457 op/s; 198/1635 objects degraded (12.110%) 2024-09-16T08:24:13.089 INFO:teuthology.orchestra.run.smithi028.stdout: 5 260550 9.36 MB/sec execute 551 sec latency 68.290 ms 2024-09-16T08:24:14.089 INFO:teuthology.orchestra.run.smithi028.stdout: 5 260868 9.37 MB/sec execute 552 sec latency 49.779 ms 2024-09-16T08:24:14.185 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:24:13 smithi028 ceph-mon[64115]: pgmap v262: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 339 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 9.5 KiB/s rd, 4.4 MiB/s wr, 441 op/s; 199/1629 objects degraded (12.216%) 2024-09-16T08:24:14.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:24:13 smithi144 ceph-mon[49149]: pgmap v262: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 339 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 9.5 KiB/s rd, 4.4 MiB/s wr, 441 op/s; 199/1629 objects degraded (12.216%) 2024-09-16T08:24:14.832 INFO:journalctl@ceph.osd.6.smithi144.stdout:Sep 16 08:24:14 smithi144 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-6[57105]: 2024-09-16T08:24:14.564+0000 7fc05d2c7640 -1 osd.6 0 read_superblock omap replica is missing. 2024-09-16T08:24:15.089 INFO:teuthology.orchestra.run.smithi028.stdout: 5 261073 9.37 MB/sec execute 553 sec latency 62.560 ms 2024-09-16T08:24:15.475 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:24:15 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:24:15.475 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:24:15 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:24:15.475 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:24:15 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-16T08:24:15.475 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:24:15 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-16T08:24:15.475 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:24:15 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:24:15.475 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:24:15 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:24:15.475 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:24:15 smithi028 ceph-mon[64115]: pgmap v263: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 337 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 4.6 MiB/s wr, 452 op/s; 206/1674 objects degraded (12.306%) 2024-09-16T08:24:15.476 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:24:15 smithi028 ceph-mon[64115]: Health check update: Degraded data redundancy: 199/1629 objects degraded (12.216%), 25 pgs degraded (PG_DEGRADED) 2024-09-16T08:24:15.476 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:24:15 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-16T08:24:15.476 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:24:15 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "versions"} : dispatch 2024-09-16T08:24:15.476 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:24:15 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "versions"} : dispatch 2024-09-16T08:24:15.476 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:24:15 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "versions"} : dispatch 2024-09-16T08:24:15.476 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:24:15 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16} : dispatch 2024-09-16T08:24:15.476 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:24:15 smithi028 ceph-mon[64115]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-09-16T08:24:15.476 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:24:15 smithi028 ceph-mon[64115]: Upgrade: unsafe to stop osd(s) at this time (11 PGs are or would become offline) 2024-09-16T08:24:15.832 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:24:15 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:24:15.832 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:24:15 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:24:15.832 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:24:15 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-16T08:24:15.832 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:24:15 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-16T08:24:15.832 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:24:15 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:24:15.832 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:24:15 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:24:15.832 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:24:15 smithi144 ceph-mon[49149]: pgmap v263: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 337 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 4.6 MiB/s wr, 452 op/s; 206/1674 objects degraded (12.306%) 2024-09-16T08:24:15.832 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:24:15 smithi144 ceph-mon[49149]: Health check update: Degraded data redundancy: 199/1629 objects degraded (12.216%), 25 pgs degraded (PG_DEGRADED) 2024-09-16T08:24:15.832 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:24:15 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-16T08:24:15.833 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:24:15 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "versions"} : dispatch 2024-09-16T08:24:15.833 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:24:15 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "versions"} : dispatch 2024-09-16T08:24:15.833 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:24:15 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "versions"} : dispatch 2024-09-16T08:24:15.833 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:24:15 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16} : dispatch 2024-09-16T08:24:15.833 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:24:15 smithi144 ceph-mon[49149]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-09-16T08:24:15.833 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:24:15 smithi144 ceph-mon[49149]: Upgrade: unsafe to stop osd(s) at this time (11 PGs are or would become offline) 2024-09-16T08:24:16.089 INFO:teuthology.orchestra.run.smithi028.stdout: 5 261352 9.37 MB/sec execute 554 sec latency 55.952 ms 2024-09-16T08:24:16.332 INFO:journalctl@ceph.osd.6.smithi144.stdout:Sep 16 08:24:15 smithi144 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-6[57105]: 2024-09-16T08:24:15.846+0000 7fc05d2c7640 -1 osd.6 136 log_to_monitors true 2024-09-16T08:24:16.545 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:24:16 smithi028 ceph-mon[64115]: from='osd.6 [v2:172.21.15.144:6816/1161923473,v1:172.21.15.144:6817/1161923473]' entity='osd.6' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]} : dispatch 2024-09-16T08:24:16.545 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:24:16 smithi028 ceph-mon[64115]: from='osd.6 ' entity='osd.6' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]} : dispatch 2024-09-16T08:24:16.831 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:24:16 smithi144 ceph-mon[49149]: from='osd.6 [v2:172.21.15.144:6816/1161923473,v1:172.21.15.144:6817/1161923473]' entity='osd.6' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]} : dispatch 2024-09-16T08:24:16.832 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:24:16 smithi144 ceph-mon[49149]: from='osd.6 ' entity='osd.6' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]} : dispatch 2024-09-16T08:24:17.089 INFO:teuthology.orchestra.run.smithi028.stdout: 5 261583 9.36 MB/sec execute 555 sec latency 170.479 ms 2024-09-16T08:24:17.332 INFO:journalctl@ceph.osd.6.smithi144.stdout:Sep 16 08:24:16 smithi144 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-6[57105]: 2024-09-16T08:24:16.887+0000 7fc051069640 -1 osd.6 136 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-16T08:24:17.560 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:24:17 smithi028 ceph-mon[64115]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-09-16T08:24:17.560 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:24:17 smithi028 ceph-mon[64115]: osdmap e139: 8 total, 7 up, 8 in 2024-09-16T08:24:17.560 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:24:17 smithi028 ceph-mon[64115]: from='osd.6 [v2:172.21.15.144:6816/1161923473,v1:172.21.15.144:6817/1161923473]' entity='osd.6' cmd={"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi144", "root=default"]} : dispatch 2024-09-16T08:24:17.560 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:24:17 smithi028 ceph-mon[64115]: from='osd.6 ' entity='osd.6' cmd={"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi144", "root=default"]} : dispatch 2024-09-16T08:24:17.561 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:24:17 smithi028 ceph-mon[64115]: from='osd.6 ' entity='osd.6' 2024-09-16T08:24:17.561 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:24:17 smithi028 ceph-mon[64115]: pgmap v265: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 340 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 6.9 KiB/s rd, 4.1 MiB/s wr, 374 op/s; 211/1689 objects degraded (12.493%) 2024-09-16T08:24:17.561 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:24:17 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:24:17.561 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:24:17 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-16T08:24:17.832 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:24:17 smithi144 ceph-mon[49149]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-09-16T08:24:17.832 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:24:17 smithi144 ceph-mon[49149]: osdmap e139: 8 total, 7 up, 8 in 2024-09-16T08:24:17.832 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:24:17 smithi144 ceph-mon[49149]: from='osd.6 [v2:172.21.15.144:6816/1161923473,v1:172.21.15.144:6817/1161923473]' entity='osd.6' cmd={"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi144", "root=default"]} : dispatch 2024-09-16T08:24:17.832 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:24:17 smithi144 ceph-mon[49149]: from='osd.6 ' entity='osd.6' cmd={"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi144", "root=default"]} : dispatch 2024-09-16T08:24:17.832 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:24:17 smithi144 ceph-mon[49149]: from='osd.6 ' entity='osd.6' 2024-09-16T08:24:17.832 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:24:17 smithi144 ceph-mon[49149]: pgmap v265: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 340 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 6.9 KiB/s rd, 4.1 MiB/s wr, 374 op/s; 211/1689 objects degraded (12.493%) 2024-09-16T08:24:17.832 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:24:17 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:24:17.832 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:24:17 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-16T08:24:18.089 INFO:teuthology.orchestra.run.smithi028.stdout: 5 261864 9.36 MB/sec execute 556 sec latency 235.208 ms 2024-09-16T08:24:18.697 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:24:18 smithi028 ceph-mon[64115]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-16T08:24:18.697 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:24:18 smithi028 ceph-mon[64115]: osd.6 [v2:172.21.15.144:6816/1161923473,v1:172.21.15.144:6817/1161923473] boot 2024-09-16T08:24:18.697 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:24:18 smithi028 ceph-mon[64115]: osdmap e140: 8 total, 8 up, 8 in 2024-09-16T08:24:18.697 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:24:18 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "osd metadata", "id": 6} : dispatch 2024-09-16T08:24:18.832 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:24:18 smithi144 ceph-mon[49149]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-16T08:24:18.832 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:24:18 smithi144 ceph-mon[49149]: osd.6 [v2:172.21.15.144:6816/1161923473,v1:172.21.15.144:6817/1161923473] boot 2024-09-16T08:24:18.832 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:24:18 smithi144 ceph-mon[49149]: osdmap e140: 8 total, 8 up, 8 in 2024-09-16T08:24:18.832 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:24:18 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "osd metadata", "id": 6} : dispatch 2024-09-16T08:24:19.089 INFO:teuthology.orchestra.run.smithi028.stdout: 5 261864 9.34 MB/sec execute 557 sec latency 1235.256 ms 2024-09-16T08:24:20.089 INFO:teuthology.orchestra.run.smithi028.stdout: 5 261890 9.32 MB/sec execute 558 sec latency 2144.731 ms 2024-09-16T08:24:20.143 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:24:19 smithi028 ceph-mon[64115]: osdmap e141: 8 total, 8 up, 8 in 2024-09-16T08:24:20.143 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:24:19 smithi028 ceph-mon[64115]: pgmap v268: 97 pgs: 11 peering, 5 active+undersized, 19 active+undersized+degraded, 62 active+clean; 344 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 4.3 KiB/s rd, 4.5 MiB/s wr, 383 op/s; 154/1692 objects degraded (9.102%) 2024-09-16T08:24:20.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:24:19 smithi144 ceph-mon[49149]: osdmap e141: 8 total, 8 up, 8 in 2024-09-16T08:24:20.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:24:19 smithi144 ceph-mon[49149]: pgmap v268: 97 pgs: 11 peering, 5 active+undersized, 19 active+undersized+degraded, 62 active+clean; 344 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 4.3 KiB/s rd, 4.5 MiB/s wr, 383 op/s; 154/1692 objects degraded (9.102%) 2024-09-16T08:24:20.947 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:24:20 smithi028 ceph-mon[64115]: osdmap e142: 8 total, 8 up, 8 in 2024-09-16T08:24:20.947 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:24:20 smithi028 ceph-mon[64115]: Health check update: Degraded data redundancy: 154/1692 objects degraded (9.102%), 19 pgs degraded (PG_DEGRADED) 2024-09-16T08:24:21.089 INFO:teuthology.orchestra.run.smithi028.stdout: 5 262074 9.31 MB/sec execute 559 sec latency 206.962 ms 2024-09-16T08:24:21.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:24:20 smithi144 ceph-mon[49149]: osdmap e142: 8 total, 8 up, 8 in 2024-09-16T08:24:21.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:24:20 smithi144 ceph-mon[49149]: Health check update: Degraded data redundancy: 154/1692 objects degraded (9.102%), 19 pgs degraded (PG_DEGRADED) 2024-09-16T08:24:22.090 INFO:teuthology.orchestra.run.smithi028.stdout: 5 262089 9.30 MB/sec execute 560 sec latency 1046.400 ms 2024-09-16T08:24:22.155 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:24:21 smithi028 ceph-mon[64115]: osdmap e143: 8 total, 8 up, 8 in 2024-09-16T08:24:22.155 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:24:21 smithi028 ceph-mon[64115]: pgmap v271: 97 pgs: 1 remapped+peering, 11 peering, 3 active+undersized, 15 active+undersized+degraded, 67 active+clean; 341 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 511 B/s rd, 3.7 MiB/s wr, 358 op/s; 119/1692 objects degraded (7.033%); 96 KiB/s, 3 objects/s recovering 2024-09-16T08:24:22.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:24:21 smithi144 ceph-mon[49149]: osdmap e143: 8 total, 8 up, 8 in 2024-09-16T08:24:22.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:24:21 smithi144 ceph-mon[49149]: pgmap v271: 97 pgs: 1 remapped+peering, 11 peering, 3 active+undersized, 15 active+undersized+degraded, 67 active+clean; 341 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 511 B/s rd, 3.7 MiB/s wr, 358 op/s; 119/1692 objects degraded (7.033%); 96 KiB/s, 3 objects/s recovering 2024-09-16T08:24:22.938 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:24:22 smithi028 ceph-mon[64115]: osdmap e144: 8 total, 8 up, 8 in 2024-09-16T08:24:23.089 INFO:teuthology.orchestra.run.smithi028.stdout: 5 262293 9.29 MB/sec execute 561 sec latency 545.714 ms 2024-09-16T08:24:23.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:24:22 smithi144 ceph-mon[49149]: osdmap e144: 8 total, 8 up, 8 in 2024-09-16T08:24:23.947 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:24:23 smithi028 ceph-mon[64115]: osdmap e145: 8 total, 8 up, 8 in 2024-09-16T08:24:23.947 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:24:23 smithi028 ceph-mon[64115]: pgmap v274: 97 pgs: 1 remapped+peering, 15 peering, 81 active+clean; 342 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 5.2 KiB/s rd, 1.6 MiB/s wr, 179 op/s; 1.8 MiB/s, 66 keys/s, 27 objects/s recovering 2024-09-16T08:24:24.089 INFO:teuthology.orchestra.run.smithi028.stdout: 5 262707 9.29 MB/sec execute 562 sec latency 37.833 ms 2024-09-16T08:24:24.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:24:23 smithi144 ceph-mon[49149]: osdmap e145: 8 total, 8 up, 8 in 2024-09-16T08:24:24.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:24:23 smithi144 ceph-mon[49149]: pgmap v274: 97 pgs: 1 remapped+peering, 15 peering, 81 active+clean; 342 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 5.2 KiB/s rd, 1.6 MiB/s wr, 179 op/s; 1.8 MiB/s, 66 keys/s, 27 objects/s recovering 2024-09-16T08:24:24.960 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:24:24 smithi028 ceph-mon[64115]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 119/1692 objects degraded (7.033%), 15 pgs degraded) 2024-09-16T08:24:24.960 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:24:24 smithi028 ceph-mon[64115]: Cluster is now healthy 2024-09-16T08:24:25.090 INFO:teuthology.orchestra.run.smithi028.stdout: 5 263126 9.29 MB/sec execute 563 sec latency 61.346 ms 2024-09-16T08:24:25.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:24:24 smithi144 ceph-mon[49149]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 119/1692 objects degraded (7.033%), 15 pgs degraded) 2024-09-16T08:24:25.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:24:24 smithi144 ceph-mon[49149]: Cluster is now healthy 2024-09-16T08:24:25.984 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:24:25 smithi028 ceph-mon[64115]: pgmap v275: 97 pgs: 4 peering, 93 active+clean; 338 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 3.8 KiB/s rd, 2.3 MiB/s wr, 278 op/s; 1.7 MiB/s, 53 keys/s, 39 objects/s recovering 2024-09-16T08:24:26.089 INFO:teuthology.orchestra.run.smithi028.stdout: 5 263433 9.29 MB/sec execute 564 sec latency 40.575 ms 2024-09-16T08:24:26.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:24:25 smithi144 ceph-mon[49149]: pgmap v275: 97 pgs: 4 peering, 93 active+clean; 338 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 3.8 KiB/s rd, 2.3 MiB/s wr, 278 op/s; 1.7 MiB/s, 53 keys/s, 39 objects/s recovering 2024-09-16T08:24:27.090 INFO:teuthology.orchestra.run.smithi028.stdout: 5 263773 9.29 MB/sec execute 565 sec latency 46.089 ms 2024-09-16T08:24:28.010 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:24:27 smithi028 ceph-mon[64115]: pgmap v276: 97 pgs: 97 active+clean; 338 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 7.3 KiB/s rd, 2.7 MiB/s wr, 339 op/s; 1.4 MiB/s, 44 keys/s, 37 objects/s recovering 2024-09-16T08:24:28.090 INFO:teuthology.orchestra.run.smithi028.stdout: 5 264156 9.29 MB/sec execute 566 sec latency 38.498 ms 2024-09-16T08:24:28.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:24:27 smithi144 ceph-mon[49149]: pgmap v276: 97 pgs: 97 active+clean; 338 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 7.3 KiB/s rd, 2.7 MiB/s wr, 339 op/s; 1.4 MiB/s, 44 keys/s, 37 objects/s recovering 2024-09-16T08:24:29.090 INFO:teuthology.orchestra.run.smithi028.stdout: 5 264462 9.29 MB/sec execute 567 sec latency 74.814 ms 2024-09-16T08:24:30.028 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:24:29 smithi028 ceph-mon[64115]: pgmap v277: 97 pgs: 97 active+clean; 336 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 5.5 KiB/s rd, 3.2 MiB/s wr, 327 op/s; 1.1 MiB/s, 33 keys/s, 27 objects/s recovering 2024-09-16T08:24:30.090 INFO:teuthology.orchestra.run.smithi028.stdout: 5 264771 9.30 MB/sec execute 568 sec latency 36.896 ms 2024-09-16T08:24:30.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:24:29 smithi144 ceph-mon[49149]: pgmap v277: 97 pgs: 97 active+clean; 336 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 5.5 KiB/s rd, 3.2 MiB/s wr, 327 op/s; 1.1 MiB/s, 33 keys/s, 27 objects/s recovering 2024-09-16T08:24:31.040 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:24:30 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16} : dispatch 2024-09-16T08:24:31.040 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:24:30 smithi028 ceph-mon[64115]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-09-16T08:24:31.040 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:24:30 smithi028 ceph-mon[64115]: Upgrade: osd.7 is safe to restart 2024-09-16T08:24:31.090 INFO:teuthology.orchestra.run.smithi028.stdout: 5 265055 9.30 MB/sec execute 569 sec latency 60.795 ms 2024-09-16T08:24:31.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:24:30 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16} : dispatch 2024-09-16T08:24:31.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:24:30 smithi144 ceph-mon[49149]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-09-16T08:24:31.333 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:24:30 smithi144 ceph-mon[49149]: Upgrade: osd.7 is safe to restart 2024-09-16T08:24:32.049 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:24:31 smithi028 ceph-mon[64115]: pgmap v278: 97 pgs: 97 active+clean; 339 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 3.0 KiB/s rd, 3.6 MiB/s wr, 345 op/s; 279 KiB/s, 29 keys/s, 17 objects/s recovering 2024-09-16T08:24:32.049 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:24:31 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:24:32.049 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:24:31 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "auth get", "entity": "osd.7"} : dispatch 2024-09-16T08:24:32.049 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:24:31 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-16T08:24:32.082 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:24:31 smithi144 ceph-mon[49149]: pgmap v278: 97 pgs: 97 active+clean; 339 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 3.0 KiB/s rd, 3.6 MiB/s wr, 345 op/s; 279 KiB/s, 29 keys/s, 17 objects/s recovering 2024-09-16T08:24:32.082 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:24:31 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:24:32.082 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:24:31 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "auth get", "entity": "osd.7"} : dispatch 2024-09-16T08:24:32.082 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:24:31 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-16T08:24:32.090 INFO:teuthology.orchestra.run.smithi028.stdout: 5 265285 9.29 MB/sec execute 570 sec latency 100.712 ms 2024-09-16T08:24:33.062 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:24:32 smithi028 ceph-mon[64115]: Upgrade: Updating osd.7 2024-09-16T08:24:33.062 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:24:32 smithi028 ceph-mon[64115]: Deploying daemon osd.7 on smithi144 2024-09-16T08:24:33.062 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:24:32 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:24:33.062 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:24:32 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-16T08:24:33.063 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:24:32 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:24:33.090 INFO:teuthology.orchestra.run.smithi028.stdout: 5 265568 9.28 MB/sec execute 571 sec latency 56.594 ms 2024-09-16T08:24:33.159 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:24:32 smithi144 ceph-mon[49149]: Upgrade: Updating osd.7 2024-09-16T08:24:33.159 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:24:32 smithi144 ceph-mon[49149]: Deploying daemon osd.7 on smithi144 2024-09-16T08:24:33.159 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:24:32 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:24:33.159 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:24:32 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-16T08:24:33.159 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:24:32 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:24:34.082 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:24:33 smithi028 ceph-mon[64115]: pgmap v279: 97 pgs: 97 active+clean; 345 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 5.5 KiB/s rd, 4.2 MiB/s wr, 426 op/s; 184 KiB/s, 12 objects/s recovering 2024-09-16T08:24:34.090 INFO:teuthology.orchestra.run.smithi028.stdout: 5 265783 9.27 MB/sec execute 572 sec latency 67.755 ms 2024-09-16T08:24:34.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:24:33 smithi144 ceph-mon[49149]: pgmap v279: 97 pgs: 97 active+clean; 345 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 5.5 KiB/s rd, 4.2 MiB/s wr, 426 op/s; 184 KiB/s, 12 objects/s recovering 2024-09-16T08:24:35.090 INFO:teuthology.orchestra.run.smithi028.stdout: 5 266084 9.28 MB/sec execute 573 sec latency 53.528 ms 2024-09-16T08:24:35.332 INFO:journalctl@ceph.osd.7.smithi144.stdout:Sep 16 08:24:35 smithi144 systemd[1]: Stopping Ceph osd.7 for 5622b2fe-7402-11ef-bceb-c7b262605968... 2024-09-16T08:24:35.832 INFO:journalctl@ceph.osd.7.smithi144.stdout:Sep 16 08:24:35 smithi144 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-7[38675]: 2024-09-16T08:24:35.484+0000 7f92abf7d700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.7 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-16T08:24:35.832 INFO:journalctl@ceph.osd.7.smithi144.stdout:Sep 16 08:24:35 smithi144 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-7[38675]: 2024-09-16T08:24:35.484+0000 7f92abf7d700 -1 osd.7 145 *** Got signal Terminated *** 2024-09-16T08:24:35.832 INFO:journalctl@ceph.osd.7.smithi144.stdout:Sep 16 08:24:35 smithi144 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-7[38675]: 2024-09-16T08:24:35.484+0000 7f92abf7d700 -1 osd.7 145 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-16T08:24:36.090 INFO:teuthology.orchestra.run.smithi028.stdout: 5 266355 9.27 MB/sec execute 574 sec latency 151.544 ms 2024-09-16T08:24:36.197 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:24:35 smithi028 ceph-mon[64115]: pgmap v280: 97 pgs: 97 active+clean; 347 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 4.6 KiB/s rd, 4.3 MiB/s wr, 429 op/s; 153 KiB/s, 10 objects/s recovering 2024-09-16T08:24:36.197 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:24:35 smithi028 ceph-mon[64115]: osd.7 marked itself down and dead 2024-09-16T08:24:36.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:24:35 smithi144 ceph-mon[49149]: pgmap v280: 97 pgs: 97 active+clean; 347 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 4.6 KiB/s rd, 4.3 MiB/s wr, 429 op/s; 153 KiB/s, 10 objects/s recovering 2024-09-16T08:24:36.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:24:35 smithi144 ceph-mon[49149]: osd.7 marked itself down and dead 2024-09-16T08:24:36.998 INFO:journalctl@ceph.osd.7.smithi144.stdout:Sep 16 08:24:36 smithi144 podman[58659]: 2024-09-16 08:24:36.648750407 +0000 UTC m=+1.302792231 container died dada74ddbe4cac43d24a61b1d3490cb88d233915daf0d01bab38d5eaeca34f60 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-7, GIT_BRANCH=HEAD, io.k8s.display-name=CentOS Stream 8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, version=8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vendor=Red Hat, Inc., io.openshift.expose-services=, maintainer=Guillaume Abrioux , distribution-scope=public, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, architecture=x86_64, com.redhat.component=centos-stream-container, vcs-type=git, release=754, ceph=True, GIT_CLEAN=True, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.license_terms=https://centos.org/legal/licensing-policy/, build-date=2022-05-03T08:36:31.336870, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.openshift.tags=base centos centos-stream, RELEASE=HEAD, name=centos-stream, GIT_REPO=https://github.com/ceph/ceph-container.git, CEPH_POINT_RELEASE=-17.2.0, io.buildah.version=1.19.8) 2024-09-16T08:24:37.090 INFO:teuthology.orchestra.run.smithi028.stdout: 5 266380 9.25 MB/sec execute 575 sec latency 1078.698 ms 2024-09-16T08:24:37.240 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:24:36 smithi028 ceph-mon[64115]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-16T08:24:37.240 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:24:36 smithi028 ceph-mon[64115]: Health check failed: all OSDs are running squid or later but require_osd_release < squid (OSD_UPGRADE_FINISHED) 2024-09-16T08:24:37.240 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:24:36 smithi028 ceph-mon[64115]: osdmap e146: 8 total, 7 up, 8 in 2024-09-16T08:24:37.240 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:24:36 smithi028 ceph-mon[64115]: osdmap e147: 8 total, 7 up, 8 in 2024-09-16T08:24:37.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:24:36 smithi144 ceph-mon[49149]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-16T08:24:37.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:24:36 smithi144 ceph-mon[49149]: Health check failed: all OSDs are running squid or later but require_osd_release < squid (OSD_UPGRADE_FINISHED) 2024-09-16T08:24:37.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:24:36 smithi144 ceph-mon[49149]: osdmap e146: 8 total, 7 up, 8 in 2024-09-16T08:24:37.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:24:36 smithi144 ceph-mon[49149]: osdmap e147: 8 total, 7 up, 8 in 2024-09-16T08:24:37.332 INFO:journalctl@ceph.osd.7.smithi144.stdout:Sep 16 08:24:37 smithi144 podman[58659]: 2024-09-16 08:24:37.144022563 +0000 UTC m=+1.798064380 container cleanup dada74ddbe4cac43d24a61b1d3490cb88d233915daf0d01bab38d5eaeca34f60 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-7, io.k8s.display-name=CentOS Stream 8, vcs-type=git, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.openshift.tags=base centos centos-stream, io.openshift.expose-services=, build-date=2022-05-03T08:36:31.336870, version=8, CEPH_POINT_RELEASE=-17.2.0, release=754, distribution-scope=public, com.redhat.component=centos-stream-container, GIT_REPO=https://github.com/ceph/ceph-container.git, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_CLEAN=True, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, ceph=True, maintainer=Guillaume Abrioux , architecture=x86_64, vendor=Red Hat, Inc., name=centos-stream, io.buildah.version=1.19.8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_BRANCH=HEAD, RELEASE=HEAD, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com) 2024-09-16T08:24:37.332 INFO:journalctl@ceph.osd.7.smithi144.stdout:Sep 16 08:24:37 smithi144 bash[58659]: ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-7 2024-09-16T08:24:37.775 INFO:journalctl@ceph.osd.7.smithi144.stdout:Sep 16 08:24:37 smithi144 podman[58674]: 2024-09-16 08:24:37.42890031 +0000 UTC m=+0.771439982 container remove dada74ddbe4cac43d24a61b1d3490cb88d233915daf0d01bab38d5eaeca34f60 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-7, release=754, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, architecture=x86_64, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=-17.2.0, GIT_BRANCH=HEAD, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.buildah.version=1.19.8, vendor=Red Hat, Inc., com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_REPO=https://github.com/ceph/ceph-container.git, ceph=True, com.redhat.component=centos-stream-container, RELEASE=HEAD, distribution-scope=public, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, vcs-type=git, version=8, name=centos-stream, io.k8s.display-name=CentOS Stream 8, io.openshift.expose-services=, GIT_CLEAN=True, io.openshift.tags=base centos centos-stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, build-date=2022-05-03T08:36:31.336870) 2024-09-16T08:24:38.077 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:24:38 smithi144 ceph-mon[49149]: pgmap v282: 97 pgs: 13 peering, 10 stale+active+clean, 74 active+clean; 343 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.6 MiB/s wr, 439 op/s 2024-09-16T08:24:38.077 INFO:journalctl@ceph.osd.7.smithi144.stdout:Sep 16 08:24:37 smithi144 podman[58779]: 2024-09-16 08:24:37.678081925 +0000 UTC m=+0.016587697 image pull 425e316ba8ebf74f04aa3640f7deb4fa3f293a78d0c8d097cb679eaf86ddca6c quay.ceph.io/ceph-ci/ceph@sha256:f7ad55f9ccd17ba66aadb19b4d291e67b60ea24e5d8a0c0e92941fd59c81e4ce 2024-09-16T08:24:38.077 INFO:journalctl@ceph.osd.7.smithi144.stdout:Sep 16 08:24:37 smithi144 podman[58779]: 2024-09-16 08:24:37.813222714 +0000 UTC m=+0.151728476 container create 3eb7b279bb27d72e15e8382a9ec7130a1ec9d001bdef9db2b184811c30205c44 (image=quay.ceph.io/ceph-ci/ceph@sha256:f7ad55f9ccd17ba66aadb19b4d291e67b60ea24e5d8a0c0e92941fd59c81e4ce, name=ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-7-deactivate, CEPH_POINT_RELEASE=, RELEASE=wip-guits-main-2024-09-13-1248-8293d73, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, ceph=True, GIT_CLEAN=True) 2024-09-16T08:24:38.077 INFO:journalctl@ceph.osd.7.smithi144.stdout:Sep 16 08:24:38 smithi144 podman[58779]: 2024-09-16 08:24:38.077283971 +0000 UTC m=+0.415789732 container init 3eb7b279bb27d72e15e8382a9ec7130a1ec9d001bdef9db2b184811c30205c44 (image=quay.ceph.io/ceph-ci/ceph@sha256:f7ad55f9ccd17ba66aadb19b4d291e67b60ea24e5d8a0c0e92941fd59c81e4ce, name=ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-7-deactivate, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , RELEASE=wip-guits-main-2024-09-13-1248-8293d73, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, ceph=True, org.label-schema.license=GPLv2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731) 2024-09-16T08:24:38.090 INFO:teuthology.orchestra.run.smithi028.stdout: 5 266761 9.25 MB/sec execute 576 sec latency 43.324 ms 2024-09-16T08:24:38.258 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:24:38 smithi028 ceph-mon[64115]: pgmap v282: 97 pgs: 13 peering, 10 stale+active+clean, 74 active+clean; 343 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.6 MiB/s wr, 439 op/s 2024-09-16T08:24:38.332 INFO:journalctl@ceph.osd.7.smithi144.stdout:Sep 16 08:24:38 smithi144 podman[58779]: 2024-09-16 08:24:38.080703405 +0000 UTC m=+0.419209174 container start 3eb7b279bb27d72e15e8382a9ec7130a1ec9d001bdef9db2b184811c30205c44 (image=quay.ceph.io/ceph-ci/ceph@sha256:f7ad55f9ccd17ba66aadb19b4d291e67b60ea24e5d8a0c0e92941fd59c81e4ce, name=ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-7-deactivate, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, ceph=True, maintainer=Guillaume Abrioux , GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, RELEASE=wip-guits-main-2024-09-13-1248-8293d73, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-16T08:24:38.333 INFO:journalctl@ceph.osd.7.smithi144.stdout:Sep 16 08:24:38 smithi144 podman[58779]: 2024-09-16 08:24:38.185751868 +0000 UTC m=+0.524257635 container attach 3eb7b279bb27d72e15e8382a9ec7130a1ec9d001bdef9db2b184811c30205c44 (image=quay.ceph.io/ceph-ci/ceph@sha256:f7ad55f9ccd17ba66aadb19b4d291e67b60ea24e5d8a0c0e92941fd59c81e4ce, name=ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-7-deactivate, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, RELEASE=wip-guits-main-2024-09-13-1248-8293d73, org.label-schema.vendor=CentOS, ceph=True) 2024-09-16T08:24:38.333 INFO:journalctl@ceph.osd.7.smithi144.stdout:Sep 16 08:24:38 smithi144 podman[58779]: 2024-09-16 08:24:38.280872445 +0000 UTC m=+0.619378204 container died 3eb7b279bb27d72e15e8382a9ec7130a1ec9d001bdef9db2b184811c30205c44 (image=quay.ceph.io/ceph-ci/ceph@sha256:f7ad55f9ccd17ba66aadb19b4d291e67b60ea24e5d8a0c0e92941fd59c81e4ce, name=ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-7-deactivate, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, RELEASE=wip-guits-main-2024-09-13-1248-8293d73, io.buildah.version=1.37.2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_CLEAN=True, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD) 2024-09-16T08:24:38.386 INFO:teuthology.orchestra.run.smithi028.stdout:true 2024-09-16T08:24:39.090 INFO:teuthology.orchestra.run.smithi028.stdout: 5 267039 9.25 MB/sec execute 577 sec latency 62.870 ms 2024-09-16T08:24:39.268 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:24:39 smithi028 ceph-mon[64115]: Health check failed: Degraded data redundancy: 103/1641 objects degraded (6.277%), 12 pgs degraded (PG_DEGRADED) 2024-09-16T08:24:39.324 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:24:39 smithi144 ceph-mon[49149]: Health check failed: Degraded data redundancy: 103/1641 objects degraded (6.277%), 12 pgs degraded (PG_DEGRADED) 2024-09-16T08:24:39.324 INFO:journalctl@ceph.osd.7.smithi144.stdout:Sep 16 08:24:38 smithi144 podman[58801]: 2024-09-16 08:24:38.882229771 +0000 UTC m=+0.595020723 container remove 3eb7b279bb27d72e15e8382a9ec7130a1ec9d001bdef9db2b184811c30205c44 (image=quay.ceph.io/ceph-ci/ceph@sha256:f7ad55f9ccd17ba66aadb19b4d291e67b60ea24e5d8a0c0e92941fd59c81e4ce, name=ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-7-deactivate, maintainer=Guillaume Abrioux , GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, ceph=True, GIT_CLEAN=True, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, RELEASE=wip-guits-main-2024-09-13-1248-8293d73, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD) 2024-09-16T08:24:39.324 INFO:journalctl@ceph.osd.7.smithi144.stdout:Sep 16 08:24:38 smithi144 systemd[1]: ceph-5622b2fe-7402-11ef-bceb-c7b262605968@osd.7.service: Deactivated successfully. 2024-09-16T08:24:39.324 INFO:journalctl@ceph.osd.7.smithi144.stdout:Sep 16 08:24:38 smithi144 systemd[1]: Stopped Ceph osd.7 for 5622b2fe-7402-11ef-bceb-c7b262605968. 2024-09-16T08:24:39.324 INFO:journalctl@ceph.osd.7.smithi144.stdout:Sep 16 08:24:38 smithi144 systemd[1]: ceph-5622b2fe-7402-11ef-bceb-c7b262605968@osd.7.service: Consumed 6min 38.581s CPU time. 2024-09-16T08:24:39.324 INFO:journalctl@ceph.osd.7.smithi144.stdout:Sep 16 08:24:39 smithi144 systemd[1]: Starting Ceph osd.7 for 5622b2fe-7402-11ef-bceb-c7b262605968... 2024-09-16T08:24:39.465 INFO:teuthology.orchestra.run.smithi028.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-16T08:24:39.466 INFO:teuthology.orchestra.run.smithi028.stdout:alertmanager.smithi028 smithi028 *:9093,9094 running (8m) 2m ago 18m 18.6M - 0.27.0 11f11916f8cd a14fcb4cb2b6 2024-09-16T08:24:39.466 INFO:teuthology.orchestra.run.smithi028.stdout:crash.smithi028 smithi028 running (5m) 2m ago 17m 6899k - 19.3.0-4917-g8293d73f 425e316ba8eb 4bb89df2a867 2024-09-16T08:24:39.466 INFO:teuthology.orchestra.run.smithi028.stdout:crash.smithi144 smithi144 running (5m) 29s ago 17m 6895k - 19.3.0-4917-g8293d73f 425e316ba8eb 24a96875d7af 2024-09-16T08:24:39.466 INFO:teuthology.orchestra.run.smithi028.stdout:grafana.smithi028 smithi028 *:3000 running (8m) 2m ago 17m 45.9M - dad864ee21e9 b55ba8b944a1 2024-09-16T08:24:39.466 INFO:teuthology.orchestra.run.smithi028.stdout:mds.foofs.smithi028.srtgev smithi028 running (12m) 2m ago 12m 38.5M - 17.2.0 e1d6a67b021e 250e7a7f754c 2024-09-16T08:24:39.466 INFO:teuthology.orchestra.run.smithi028.stdout:mds.foofs.smithi144.oozgsi smithi144 running (12m) 29s ago 12m 17.6M - 17.2.0 e1d6a67b021e ecf4c2bb0fdc 2024-09-16T08:24:39.466 INFO:teuthology.orchestra.run.smithi028.stdout:mgr.smithi028.azvyzy smithi028 *:8443,9283,8765 running (8m) 2m ago 19m 519M - 19.3.0-4917-g8293d73f 425e316ba8eb e01d4382d03f 2024-09-16T08:24:39.466 INFO:teuthology.orchestra.run.smithi028.stdout:mgr.smithi144.utzszj smithi144 *:8443,9283,8765 running (7m) 29s ago 17m 457M - 19.3.0-4917-g8293d73f 425e316ba8eb 71f7f802bffb 2024-09-16T08:24:39.466 INFO:teuthology.orchestra.run.smithi028.stdout:mon.smithi028 smithi028 running (7m) 2m ago 19m 49.8M 2048M 19.3.0-4917-g8293d73f 425e316ba8eb 14efcedc0088 2024-09-16T08:24:39.466 INFO:teuthology.orchestra.run.smithi028.stdout:mon.smithi144 smithi144 running (6m) 29s ago 17m 49.6M 2048M 19.3.0-4917-g8293d73f 425e316ba8eb c081c5ed44b5 2024-09-16T08:24:39.467 INFO:teuthology.orchestra.run.smithi028.stdout:nfs.foo.0.0.smithi028.dkttgj smithi028 *:2049 running (11m) 2m ago 11m 285M - 4.0 e1d6a67b021e b5dccc45a6ed 2024-09-16T08:24:39.467 INFO:teuthology.orchestra.run.smithi028.stdout:nfs.foo.1.0.smithi144.epipkg smithi144 *:2049 running (11m) 29s ago 11m 87.4M - 4.0 e1d6a67b021e faa89e61f50e 2024-09-16T08:24:39.467 INFO:teuthology.orchestra.run.smithi028.stdout:node-exporter.smithi028 smithi028 *:9100 running (8m) 2m ago 17m 9.92M - 1.7.0 72c9c2088986 9e2b8e8a63a9 2024-09-16T08:24:39.467 INFO:teuthology.orchestra.run.smithi028.stdout:node-exporter.smithi144 smithi144 *:9100 running (8m) 29s ago 17m 10.9M - 1.7.0 72c9c2088986 78834c5e992f 2024-09-16T08:24:39.467 INFO:teuthology.orchestra.run.smithi028.stdout:osd.0 smithi028 running (5m) 2m ago 16m 354M 1937M 19.3.0-4917-g8293d73f 425e316ba8eb 2fc02ba4f14f 2024-09-16T08:24:39.467 INFO:teuthology.orchestra.run.smithi028.stdout:osd.1 smithi028 running (4m) 2m ago 16m 266M 1937M 19.3.0-4917-g8293d73f 425e316ba8eb df34bd603d8f 2024-09-16T08:24:39.467 INFO:teuthology.orchestra.run.smithi028.stdout:osd.2 smithi028 running (3m) 2m ago 15m 220M 1937M 19.3.0-4917-g8293d73f 425e316ba8eb c92be6327b12 2024-09-16T08:24:39.467 INFO:teuthology.orchestra.run.smithi028.stdout:osd.3 smithi028 running (2m) 2m ago 15m 11.2M 1937M 19.3.0-4917-g8293d73f 425e316ba8eb bd84ba1224f2 2024-09-16T08:24:39.467 INFO:teuthology.orchestra.run.smithi028.stdout:osd.4 smithi144 running (111s) 29s ago 14m 310M 2704M 19.3.0-4917-g8293d73f 425e316ba8eb c48cb68e901e 2024-09-16T08:24:39.467 INFO:teuthology.orchestra.run.smithi028.stdout:osd.5 smithi144 running (73s) 29s ago 14m 258M 2704M 19.3.0-4917-g8293d73f 425e316ba8eb 59ec97fb3cf3 2024-09-16T08:24:39.467 INFO:teuthology.orchestra.run.smithi028.stdout:osd.6 smithi144 running (34s) 29s ago 14m 11.2M 2704M 19.3.0-4917-g8293d73f 425e316ba8eb b301bf0731f0 2024-09-16T08:24:39.467 INFO:teuthology.orchestra.run.smithi028.stdout:osd.7 smithi144 running (13m) 29s ago 13m 809M 2704M 17.2.0 e1d6a67b021e dada74ddbe4c 2024-09-16T08:24:39.467 INFO:teuthology.orchestra.run.smithi028.stdout:prometheus.smithi028 smithi028 *:9095 running (7m) 2m ago 16m 55.3M - 2.51.0 1d3b7f56885b 42667809df2b 2024-09-16T08:24:39.582 INFO:journalctl@ceph.osd.7.smithi144.stdout:Sep 16 08:24:39 smithi144 podman[58902]: 2024-09-16 08:24:39.226871741 +0000 UTC m=+0.017972549 image pull 425e316ba8ebf74f04aa3640f7deb4fa3f293a78d0c8d097cb679eaf86ddca6c quay.ceph.io/ceph-ci/ceph@sha256:f7ad55f9ccd17ba66aadb19b4d291e67b60ea24e5d8a0c0e92941fd59c81e4ce 2024-09-16T08:24:39.582 INFO:journalctl@ceph.osd.7.smithi144.stdout:Sep 16 08:24:39 smithi144 podman[58902]: 2024-09-16 08:24:39.350415951 +0000 UTC m=+0.141516752 container create 4c6980b77f5e94920fa0c2a0806fa95beac71b88522150001e854d129d7299b5 (image=quay.ceph.io/ceph-ci/ceph@sha256:f7ad55f9ccd17ba66aadb19b4d291e67b60ea24e5d8a0c0e92941fd59c81e4ce, name=ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-7-activate, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, RELEASE=wip-guits-main-2024-09-13-1248-8293d73, org.label-schema.license=GPLv2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965) 2024-09-16T08:24:40.033 INFO:teuthology.orchestra.run.smithi028.stdout:{ 2024-09-16T08:24:40.034 INFO:teuthology.orchestra.run.smithi028.stdout: "mon": { 2024-09-16T08:24:40.034 INFO:teuthology.orchestra.run.smithi028.stdout: "ceph version 19.3.0-4917-g8293d73f (8293d73f8690540e843a81caec373f9cc29cf705) squid (dev)": 2 2024-09-16T08:24:40.034 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-09-16T08:24:40.034 INFO:teuthology.orchestra.run.smithi028.stdout: "mgr": { 2024-09-16T08:24:40.034 INFO:teuthology.orchestra.run.smithi028.stdout: "ceph version 19.3.0-4917-g8293d73f (8293d73f8690540e843a81caec373f9cc29cf705) squid (dev)": 2 2024-09-16T08:24:40.034 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-09-16T08:24:40.034 INFO:teuthology.orchestra.run.smithi028.stdout: "osd": { 2024-09-16T08:24:40.034 INFO:teuthology.orchestra.run.smithi028.stdout: "ceph version 19.3.0-4917-g8293d73f (8293d73f8690540e843a81caec373f9cc29cf705) squid (dev)": 7 2024-09-16T08:24:40.034 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-09-16T08:24:40.034 INFO:teuthology.orchestra.run.smithi028.stdout: "mds": { 2024-09-16T08:24:40.034 INFO:teuthology.orchestra.run.smithi028.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-16T08:24:40.034 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-09-16T08:24:40.034 INFO:teuthology.orchestra.run.smithi028.stdout: "overall": { 2024-09-16T08:24:40.035 INFO:teuthology.orchestra.run.smithi028.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2, 2024-09-16T08:24:40.035 INFO:teuthology.orchestra.run.smithi028.stdout: "ceph version 19.3.0-4917-g8293d73f (8293d73f8690540e843a81caec373f9cc29cf705) squid (dev)": 11 2024-09-16T08:24:40.035 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-09-16T08:24:40.035 INFO:teuthology.orchestra.run.smithi028.stdout:} 2024-09-16T08:24:40.069 INFO:journalctl@ceph.osd.7.smithi144.stdout:Sep 16 08:24:39 smithi144 podman[58902]: 2024-09-16 08:24:39.62924253 +0000 UTC m=+0.420343328 container init 4c6980b77f5e94920fa0c2a0806fa95beac71b88522150001e854d129d7299b5 (image=quay.ceph.io/ceph-ci/ceph@sha256:f7ad55f9ccd17ba66aadb19b4d291e67b60ea24e5d8a0c0e92941fd59c81e4ce, name=ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-7-activate, org.label-schema.license=GPLv2, RELEASE=wip-guits-main-2024-09-13-1248-8293d73, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-16T08:24:40.069 INFO:journalctl@ceph.osd.7.smithi144.stdout:Sep 16 08:24:39 smithi144 podman[58902]: 2024-09-16 08:24:39.634819855 +0000 UTC m=+0.425920653 container start 4c6980b77f5e94920fa0c2a0806fa95beac71b88522150001e854d129d7299b5 (image=quay.ceph.io/ceph-ci/ceph@sha256:f7ad55f9ccd17ba66aadb19b4d291e67b60ea24e5d8a0c0e92941fd59c81e4ce, name=ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-7-activate, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, RELEASE=wip-guits-main-2024-09-13-1248-8293d73, maintainer=Guillaume Abrioux ) 2024-09-16T08:24:40.069 INFO:journalctl@ceph.osd.7.smithi144.stdout:Sep 16 08:24:39 smithi144 podman[58902]: 2024-09-16 08:24:39.72933233 +0000 UTC m=+0.520433129 container attach 4c6980b77f5e94920fa0c2a0806fa95beac71b88522150001e854d129d7299b5 (image=quay.ceph.io/ceph-ci/ceph@sha256:f7ad55f9ccd17ba66aadb19b4d291e67b60ea24e5d8a0c0e92941fd59c81e4ce, name=ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-7-activate, GIT_CLEAN=True, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, RELEASE=wip-guits-main-2024-09-13-1248-8293d73, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=) 2024-09-16T08:24:40.070 INFO:journalctl@ceph.osd.7.smithi144.stdout:Sep 16 08:24:39 smithi144 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-7-activate[58917]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-16T08:24:40.070 INFO:journalctl@ceph.osd.7.smithi144.stdout:Sep 16 08:24:39 smithi144 bash[58902]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-16T08:24:40.070 INFO:journalctl@ceph.osd.7.smithi144.stdout:Sep 16 08:24:39 smithi144 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-7-activate[58917]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-16T08:24:40.070 INFO:journalctl@ceph.osd.7.smithi144.stdout:Sep 16 08:24:39 smithi144 bash[58902]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-16T08:24:40.090 INFO:teuthology.orchestra.run.smithi028.stdout: 5 267325 9.25 MB/sec execute 578 sec latency 50.863 ms 2024-09-16T08:24:40.285 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:24:40 smithi028 ceph-mon[64115]: from='client.34430 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T08:24:40.285 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:24:40 smithi028 ceph-mon[64115]: from='client.44373 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T08:24:40.285 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:24:40 smithi028 ceph-mon[64115]: pgmap v284: 97 pgs: 13 peering, 4 stale+active+clean, 12 active+undersized+degraded, 68 active+clean; 344 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 8.1 KiB/s rd, 4.4 MiB/s wr, 466 op/s; 103/1641 objects degraded (6.277%) 2024-09-16T08:24:40.285 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:24:40 smithi028 ceph-mon[64115]: from='client.? 172.21.15.28:0/2003041161' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-16T08:24:40.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:24:40 smithi144 ceph-mon[49149]: from='client.34430 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T08:24:40.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:24:40 smithi144 ceph-mon[49149]: from='client.44373 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T08:24:40.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:24:40 smithi144 ceph-mon[49149]: pgmap v284: 97 pgs: 13 peering, 4 stale+active+clean, 12 active+undersized+degraded, 68 active+clean; 344 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 8.1 KiB/s rd, 4.4 MiB/s wr, 466 op/s; 103/1641 objects degraded (6.277%) 2024-09-16T08:24:40.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:24:40 smithi144 ceph-mon[49149]: from='client.? 172.21.15.28:0/2003041161' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-16T08:24:40.531 INFO:teuthology.orchestra.run.smithi028.stdout:{ 2024-09-16T08:24:40.531 INFO:teuthology.orchestra.run.smithi028.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:f7ad55f9ccd17ba66aadb19b4d291e67b60ea24e5d8a0c0e92941fd59c81e4ce", 2024-09-16T08:24:40.531 INFO:teuthology.orchestra.run.smithi028.stdout: "in_progress": true, 2024-09-16T08:24:40.531 INFO:teuthology.orchestra.run.smithi028.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-16T08:24:40.531 INFO:teuthology.orchestra.run.smithi028.stdout: "services_complete": [ 2024-09-16T08:24:40.532 INFO:teuthology.orchestra.run.smithi028.stdout: "crash", 2024-09-16T08:24:40.532 INFO:teuthology.orchestra.run.smithi028.stdout: "mon", 2024-09-16T08:24:40.532 INFO:teuthology.orchestra.run.smithi028.stdout: "mgr" 2024-09-16T08:24:40.532 INFO:teuthology.orchestra.run.smithi028.stdout: ], 2024-09-16T08:24:40.532 INFO:teuthology.orchestra.run.smithi028.stdout: "progress": "13/23 daemons upgraded", 2024-09-16T08:24:40.532 INFO:teuthology.orchestra.run.smithi028.stdout: "message": "Currently upgrading osd daemons", 2024-09-16T08:24:40.533 INFO:teuthology.orchestra.run.smithi028.stdout: "is_paused": false 2024-09-16T08:24:40.533 INFO:teuthology.orchestra.run.smithi028.stdout:} 2024-09-16T08:24:41.090 INFO:teuthology.orchestra.run.smithi028.stdout: 5 267756 9.25 MB/sec execute 579 sec latency 47.277 ms 2024-09-16T08:24:41.205 INFO:teuthology.orchestra.run.smithi028.stdout:HEALTH_WARN 1 osds down; all OSDs are running squid or later but require_osd_release < squid; Degraded data redundancy: 190/1626 objects degraded (11.685%), 24 pgs degraded 2024-09-16T08:24:41.205 INFO:teuthology.orchestra.run.smithi028.stdout:[WRN] OSD_DOWN: 1 osds down 2024-09-16T08:24:41.205 INFO:teuthology.orchestra.run.smithi028.stdout: osd.7 (root=default,host=smithi144) is down 2024-09-16T08:24:41.205 INFO:teuthology.orchestra.run.smithi028.stdout:[WRN] OSD_UPGRADE_FINISHED: all OSDs are running squid or later but require_osd_release < squid 2024-09-16T08:24:41.206 INFO:teuthology.orchestra.run.smithi028.stdout: all OSDs are running squid or later but require_osd_release < squid 2024-09-16T08:24:41.206 INFO:teuthology.orchestra.run.smithi028.stdout:[WRN] PG_DEGRADED: Degraded data redundancy: 190/1626 objects degraded (11.685%), 24 pgs degraded 2024-09-16T08:24:41.206 INFO:teuthology.orchestra.run.smithi028.stdout: pg 1.0 is active+undersized+degraded, acting [0,6] 2024-09-16T08:24:41.206 INFO:teuthology.orchestra.run.smithi028.stdout: pg 2.0 is active+undersized+degraded, acting [1,0] 2024-09-16T08:24:41.206 INFO:teuthology.orchestra.run.smithi028.stdout: pg 2.4 is active+undersized+degraded, acting [1,0] 2024-09-16T08:24:41.206 INFO:teuthology.orchestra.run.smithi028.stdout: pg 2.5 is active+undersized+degraded, acting [0,4] 2024-09-16T08:24:41.206 INFO:teuthology.orchestra.run.smithi028.stdout: pg 2.8 is active+undersized+degraded, acting [0,1] 2024-09-16T08:24:41.206 INFO:teuthology.orchestra.run.smithi028.stdout: pg 2.9 is active+undersized+degraded, acting [1,3] 2024-09-16T08:24:41.206 INFO:teuthology.orchestra.run.smithi028.stdout: pg 2.a is active+undersized+degraded, acting [1,3] 2024-09-16T08:24:41.206 INFO:teuthology.orchestra.run.smithi028.stdout: pg 2.b is active+undersized+degraded, acting [4,5] 2024-09-16T08:24:41.206 INFO:teuthology.orchestra.run.smithi028.stdout: pg 2.e is active+undersized+degraded, acting [2,3] 2024-09-16T08:24:41.206 INFO:teuthology.orchestra.run.smithi028.stdout: pg 2.f is active+undersized+degraded, acting [4,0] 2024-09-16T08:24:41.206 INFO:teuthology.orchestra.run.smithi028.stdout: pg 2.1f is active+undersized+degraded, acting [0,4] 2024-09-16T08:24:41.206 INFO:teuthology.orchestra.run.smithi028.stdout: pg 3.7 is active+undersized+degraded, acting [3,0] 2024-09-16T08:24:41.207 INFO:teuthology.orchestra.run.smithi028.stdout: pg 3.8 is active+undersized+degraded, acting [3,1] 2024-09-16T08:24:41.207 INFO:teuthology.orchestra.run.smithi028.stdout: pg 3.9 is active+undersized+degraded, acting [4,2] 2024-09-16T08:24:41.207 INFO:teuthology.orchestra.run.smithi028.stdout: pg 3.e is active+undersized+degraded, acting [4,1] 2024-09-16T08:24:41.207 INFO:teuthology.orchestra.run.smithi028.stdout: pg 3.f is active+undersized+degraded, acting [4,0] 2024-09-16T08:24:41.207 INFO:teuthology.orchestra.run.smithi028.stdout: pg 3.11 is active+undersized+degraded, acting [4,6] 2024-09-16T08:24:41.207 INFO:teuthology.orchestra.run.smithi028.stdout: pg 3.12 is active+undersized+degraded, acting [0,3] 2024-09-16T08:24:41.207 INFO:teuthology.orchestra.run.smithi028.stdout: pg 3.13 is active+undersized+degraded, acting [4,2] 2024-09-16T08:24:41.207 INFO:teuthology.orchestra.run.smithi028.stdout: pg 3.14 is active+undersized+degraded, acting [4,6] 2024-09-16T08:24:41.207 INFO:teuthology.orchestra.run.smithi028.stdout: pg 3.15 is active+undersized+degraded, acting [3,4] 2024-09-16T08:24:41.207 INFO:teuthology.orchestra.run.smithi028.stdout: pg 3.1b is active+undersized+degraded, acting [0,4] 2024-09-16T08:24:41.207 INFO:teuthology.orchestra.run.smithi028.stdout: pg 4.0 is active+undersized+degraded, acting [3,0] 2024-09-16T08:24:41.207 INFO:teuthology.orchestra.run.smithi028.stdout: pg 4.14 is active+undersized+degraded, acting [3,1] 2024-09-16T08:24:41.297 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:24:41 smithi028 ceph-mon[64115]: from='client.34438 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T08:24:41.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:24:41 smithi144 ceph-mon[49149]: from='client.34438 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T08:24:42.090 INFO:teuthology.orchestra.run.smithi028.stdout: 5 268144 9.26 MB/sec execute 580 sec latency 63.058 ms 2024-09-16T08:24:42.320 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:24:42 smithi028 ceph-mon[64115]: from='client.34446 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T08:24:42.320 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:24:42 smithi028 ceph-mon[64115]: pgmap v285: 97 pgs: 5 active+undersized, 13 peering, 24 active+undersized+degraded, 55 active+clean; 341 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 3.7 MiB/s wr, 352 op/s; 190/1626 objects degraded (11.685%) 2024-09-16T08:24:42.320 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:24:42 smithi028 ceph-mon[64115]: from='client.? 172.21.15.28:0/2251911364' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-16T08:24:42.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:24:42 smithi144 ceph-mon[49149]: from='client.34446 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T08:24:42.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:24:42 smithi144 ceph-mon[49149]: pgmap v285: 97 pgs: 5 active+undersized, 13 peering, 24 active+undersized+degraded, 55 active+clean; 341 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 3.7 MiB/s wr, 352 op/s; 190/1626 objects degraded (11.685%) 2024-09-16T08:24:42.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:24:42 smithi144 ceph-mon[49149]: from='client.? 172.21.15.28:0/2251911364' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-16T08:24:42.333 INFO:journalctl@ceph.osd.7.smithi144.stdout:Sep 16 08:24:42 smithi144 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-7-activate[58917]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-7 2024-09-16T08:24:42.333 INFO:journalctl@ceph.osd.7.smithi144.stdout:Sep 16 08:24:42 smithi144 bash[58902]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-7 2024-09-16T08:24:42.333 INFO:journalctl@ceph.osd.7.smithi144.stdout:Sep 16 08:24:42 smithi144 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-7-activate[58917]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-7 --no-mon-config --dev /dev/mapper/vg_nvme-lv_1 2024-09-16T08:24:42.333 INFO:journalctl@ceph.osd.7.smithi144.stdout:Sep 16 08:24:42 smithi144 bash[58902]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-7 --no-mon-config --dev /dev/mapper/vg_nvme-lv_1 2024-09-16T08:24:42.767 INFO:journalctl@ceph.osd.7.smithi144.stdout:Sep 16 08:24:42 smithi144 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-7-activate[58917]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_1 2024-09-16T08:24:42.767 INFO:journalctl@ceph.osd.7.smithi144.stdout:Sep 16 08:24:42 smithi144 bash[58902]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_1 2024-09-16T08:24:42.767 INFO:journalctl@ceph.osd.7.smithi144.stdout:Sep 16 08:24:42 smithi144 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-7-activate[58917]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-09-16T08:24:42.767 INFO:journalctl@ceph.osd.7.smithi144.stdout:Sep 16 08:24:42 smithi144 bash[58902]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-09-16T08:24:42.768 INFO:journalctl@ceph.osd.7.smithi144.stdout:Sep 16 08:24:42 smithi144 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-7-activate[58917]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-7/block 2024-09-16T08:24:42.768 INFO:journalctl@ceph.osd.7.smithi144.stdout:Sep 16 08:24:42 smithi144 bash[58902]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-7/block 2024-09-16T08:24:42.768 INFO:journalctl@ceph.osd.7.smithi144.stdout:Sep 16 08:24:42 smithi144 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-7-activate[58917]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-7 2024-09-16T08:24:42.768 INFO:journalctl@ceph.osd.7.smithi144.stdout:Sep 16 08:24:42 smithi144 bash[58902]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-7 2024-09-16T08:24:42.768 INFO:journalctl@ceph.osd.7.smithi144.stdout:Sep 16 08:24:42 smithi144 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-7-activate[58917]: --> ceph-volume raw activate successful for osd ID: 7 2024-09-16T08:24:42.768 INFO:journalctl@ceph.osd.7.smithi144.stdout:Sep 16 08:24:42 smithi144 bash[58902]: --> ceph-volume raw activate successful for osd ID: 7 2024-09-16T08:24:42.768 INFO:journalctl@ceph.osd.7.smithi144.stdout:Sep 16 08:24:42 smithi144 podman[58902]: 2024-09-16 08:24:42.453456826 +0000 UTC m=+3.244557627 container died 4c6980b77f5e94920fa0c2a0806fa95beac71b88522150001e854d129d7299b5 (image=quay.ceph.io/ceph-ci/ceph@sha256:f7ad55f9ccd17ba66aadb19b4d291e67b60ea24e5d8a0c0e92941fd59c81e4ce, name=ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-7-activate, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.vendor=CentOS, ceph=True, RELEASE=wip-guits-main-2024-09-13-1248-8293d73, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-16T08:24:43.090 INFO:teuthology.orchestra.run.smithi028.stdout: 5 268507 9.26 MB/sec execute 581 sec latency 34.941 ms 2024-09-16T08:24:43.332 INFO:journalctl@ceph.osd.7.smithi144.stdout:Sep 16 08:24:43 smithi144 podman[59074]: 2024-09-16 08:24:43.076282964 +0000 UTC m=+0.616405148 container remove 4c6980b77f5e94920fa0c2a0806fa95beac71b88522150001e854d129d7299b5 (image=quay.ceph.io/ceph-ci/ceph@sha256:f7ad55f9ccd17ba66aadb19b4d291e67b60ea24e5d8a0c0e92941fd59c81e4ce, name=ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-7-activate, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, GIT_CLEAN=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, RELEASE=wip-guits-main-2024-09-13-1248-8293d73, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, ceph=True, maintainer=Guillaume Abrioux ) 2024-09-16T08:24:43.332 INFO:journalctl@ceph.osd.7.smithi144.stdout:Sep 16 08:24:43 smithi144 podman[59131]: 2024-09-16 08:24:43.194224637 +0000 UTC m=+0.018481593 image pull 425e316ba8ebf74f04aa3640f7deb4fa3f293a78d0c8d097cb679eaf86ddca6c quay.ceph.io/ceph-ci/ceph@sha256:f7ad55f9ccd17ba66aadb19b4d291e67b60ea24e5d8a0c0e92941fd59c81e4ce 2024-09-16T08:24:43.332 INFO:journalctl@ceph.osd.7.smithi144.stdout:Sep 16 08:24:43 smithi144 podman[59131]: 2024-09-16 08:24:43.31906158 +0000 UTC m=+0.143318535 container create f1a5cbf3a7bd4507c6d1a99600de331ca98754dd71c18ffb754e85900c30bd5e (image=quay.ceph.io/ceph-ci/ceph@sha256:f7ad55f9ccd17ba66aadb19b4d291e67b60ea24e5d8a0c0e92941fd59c81e4ce, name=ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-7, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_CLEAN=True, RELEASE=wip-guits-main-2024-09-13-1248-8293d73, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS) 2024-09-16T08:24:43.722 INFO:journalctl@ceph.osd.7.smithi144.stdout:Sep 16 08:24:43 smithi144 podman[59131]: 2024-09-16 08:24:43.622430392 +0000 UTC m=+0.446687348 container init f1a5cbf3a7bd4507c6d1a99600de331ca98754dd71c18ffb754e85900c30bd5e (image=quay.ceph.io/ceph-ci/ceph@sha256:f7ad55f9ccd17ba66aadb19b4d291e67b60ea24e5d8a0c0e92941fd59c81e4ce, name=ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-7, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=wip-guits-main-2024-09-13-1248-8293d73, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD) 2024-09-16T08:24:43.722 INFO:journalctl@ceph.osd.7.smithi144.stdout:Sep 16 08:24:43 smithi144 podman[59131]: 2024-09-16 08:24:43.626416187 +0000 UTC m=+0.450673144 container start f1a5cbf3a7bd4507c6d1a99600de331ca98754dd71c18ffb754e85900c30bd5e (image=quay.ceph.io/ceph-ci/ceph@sha256:f7ad55f9ccd17ba66aadb19b4d291e67b60ea24e5d8a0c0e92941fd59c81e4ce, name=ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-7, GIT_CLEAN=True, org.label-schema.vendor=CentOS, RELEASE=wip-guits-main-2024-09-13-1248-8293d73, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, ceph=True, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2) 2024-09-16T08:24:43.722 INFO:journalctl@ceph.osd.7.smithi144.stdout:Sep 16 08:24:43 smithi144 ceph-osd[59150]: -- 172.21.15.144:0/1860597813 <== mon.1 v2:172.21.15.144:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x5561662f61e0 con 0x556166288000 2024-09-16T08:24:44.077 INFO:journalctl@ceph.osd.7.smithi144.stdout:Sep 16 08:24:43 smithi144 bash[59131]: f1a5cbf3a7bd4507c6d1a99600de331ca98754dd71c18ffb754e85900c30bd5e 2024-09-16T08:24:44.077 INFO:journalctl@ceph.osd.7.smithi144.stdout:Sep 16 08:24:43 smithi144 systemd[1]: Started Ceph osd.7 for 5622b2fe-7402-11ef-bceb-c7b262605968. 2024-09-16T08:24:44.090 INFO:teuthology.orchestra.run.smithi028.stdout: 5 268840 9.26 MB/sec execute 582 sec latency 46.017 ms 2024-09-16T08:24:44.095 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:24:44 smithi028 ceph-mon[64115]: pgmap v286: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 342 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 5.9 KiB/s rd, 3.8 MiB/s wr, 436 op/s; 253/1656 objects degraded (15.278%) 2024-09-16T08:24:44.095 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:24:44 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:24:44.096 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:24:44 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:24:44.096 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:24:44 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-16T08:24:44.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:24:44 smithi144 ceph-mon[49149]: pgmap v286: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 342 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 5.9 KiB/s rd, 3.8 MiB/s wr, 436 op/s; 253/1656 objects degraded (15.278%) 2024-09-16T08:24:44.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:24:44 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:24:44.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:24:44 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:24:44.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:24:44 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-16T08:24:45.090 INFO:teuthology.orchestra.run.smithi028.stdout: 5 269128 9.25 MB/sec execute 583 sec latency 43.715 ms 2024-09-16T08:24:45.354 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:24:45 smithi028 ceph-mon[64115]: Health check update: Degraded data redundancy: 253/1656 objects degraded (15.278%), 35 pgs degraded (PG_DEGRADED) 2024-09-16T08:24:45.516 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:24:45 smithi144 ceph-mon[49149]: Health check update: Degraded data redundancy: 253/1656 objects degraded (15.278%), 35 pgs degraded (PG_DEGRADED) 2024-09-16T08:24:45.516 INFO:journalctl@ceph.osd.7.smithi144.stdout:Sep 16 08:24:45 smithi144 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-7[59146]: 2024-09-16T08:24:45.269+0000 7f4aa14a0640 -1 Falling back to public interface 2024-09-16T08:24:46.090 INFO:teuthology.orchestra.run.smithi028.stdout: 5 269345 9.25 MB/sec execute 584 sec latency 52.833 ms 2024-09-16T08:24:46.114 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:24:46 smithi028 ceph-mon[64115]: pgmap v287: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 348 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 5.2 KiB/s rd, 4.6 MiB/s wr, 474 op/s; 261/1698 objects degraded (15.371%) 2024-09-16T08:24:46.469 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:24:46 smithi144 ceph-mon[49149]: pgmap v287: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 348 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 5.2 KiB/s rd, 4.6 MiB/s wr, 474 op/s; 261/1698 objects degraded (15.371%) 2024-09-16T08:24:47.090 INFO:teuthology.orchestra.run.smithi028.stdout: 5 269662 9.25 MB/sec execute 585 sec latency 45.570 ms 2024-09-16T08:24:48.090 INFO:teuthology.orchestra.run.smithi028.stdout: 5 269970 9.24 MB/sec execute 586 sec latency 52.693 ms 2024-09-16T08:24:48.362 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:24:48 smithi028 ceph-mon[64115]: pgmap v288: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 348 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 5.0 KiB/s rd, 4.7 MiB/s wr, 437 op/s; 268/1719 objects degraded (15.590%) 2024-09-16T08:24:48.362 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:24:48 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:24:48.362 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:24:48 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-16T08:24:48.363 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:24:48 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:24:48.363 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:24:48 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:24:48.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:24:48 smithi144 ceph-mon[49149]: pgmap v288: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 348 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 5.0 KiB/s rd, 4.7 MiB/s wr, 437 op/s; 268/1719 objects degraded (15.590%) 2024-09-16T08:24:48.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:24:48 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:24:48.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:24:48 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-16T08:24:48.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:24:48 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:24:48.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:24:48 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:24:49.090 INFO:teuthology.orchestra.run.smithi028.stdout: 5 270318 9.24 MB/sec execute 587 sec latency 57.988 ms 2024-09-16T08:24:49.420 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:24:49 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:24:49.420 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:24:49 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:24:49.420 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:24:49 smithi028 ceph-mon[64115]: pgmap v289: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 345 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 4.2 KiB/s rd, 4.3 MiB/s wr, 390 op/s; 252/1671 objects degraded (15.081%) 2024-09-16T08:24:49.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:24:49 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:24:49.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:24:49 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:24:49.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:24:49 smithi144 ceph-mon[49149]: pgmap v289: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 345 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 4.2 KiB/s rd, 4.3 MiB/s wr, 390 op/s; 252/1671 objects degraded (15.081%) 2024-09-16T08:24:50.090 INFO:teuthology.orchestra.run.smithi028.stdout: 5 270602 9.23 MB/sec execute 588 sec latency 51.640 ms 2024-09-16T08:24:50.466 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:24:50 smithi028 ceph-mon[64115]: Health check update: Degraded data redundancy: 252/1671 objects degraded (15.081%), 35 pgs degraded (PG_DEGRADED) 2024-09-16T08:24:50.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:24:50 smithi144 ceph-mon[49149]: Health check update: Degraded data redundancy: 252/1671 objects degraded (15.081%), 35 pgs degraded (PG_DEGRADED) 2024-09-16T08:24:51.090 INFO:teuthology.orchestra.run.smithi028.stdout: 5 270913 9.24 MB/sec execute 589 sec latency 64.530 ms 2024-09-16T08:24:51.510 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:24:51 smithi028 ceph-mon[64115]: pgmap v290: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 345 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 4.3 KiB/s rd, 4.6 MiB/s wr, 466 op/s; 260/1680 objects degraded (15.476%) 2024-09-16T08:24:51.704 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:24:51 smithi144 ceph-mon[49149]: pgmap v290: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 345 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 4.3 KiB/s rd, 4.6 MiB/s wr, 466 op/s; 260/1680 objects degraded (15.476%) 2024-09-16T08:24:52.090 INFO:teuthology.orchestra.run.smithi028.stdout: 5 271282 9.24 MB/sec execute 590 sec latency 56.465 ms 2024-09-16T08:24:52.792 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:24:52 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:24:52.792 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:24:52 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:24:52.792 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:24:52 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-16T08:24:52.792 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:24:52 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-16T08:24:52.792 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:24:52 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:24:52.792 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:24:52 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:24:52.793 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:24:52 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-16T08:24:52.793 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:24:52 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "versions"} : dispatch 2024-09-16T08:24:52.793 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:24:52 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "versions"} : dispatch 2024-09-16T08:24:52.793 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:24:52 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "versions"} : dispatch 2024-09-16T08:24:52.793 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:24:52 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "versions"} : dispatch 2024-09-16T08:24:52.793 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:24:52 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:24:52.793 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:24:52 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.0"} : dispatch 2024-09-16T08:24:52.793 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:24:52 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.0"}]': finished 2024-09-16T08:24:52.793 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:24:52 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.1"} : dispatch 2024-09-16T08:24:52.793 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:24:52 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.1"}]': finished 2024-09-16T08:24:52.793 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:24:52 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.2"} : dispatch 2024-09-16T08:24:52.793 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:24:52 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.2"}]': finished 2024-09-16T08:24:52.793 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:24:52 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.3"} : dispatch 2024-09-16T08:24:52.794 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:24:52 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.3"}]': finished 2024-09-16T08:24:52.794 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:24:52 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.4"} : dispatch 2024-09-16T08:24:52.794 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:24:52 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.4"}]': finished 2024-09-16T08:24:52.794 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:24:52 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.5"} : dispatch 2024-09-16T08:24:52.794 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:24:52 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.5"}]': finished 2024-09-16T08:24:52.794 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:24:52 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.6"} : dispatch 2024-09-16T08:24:52.794 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:24:52 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.6"}]': finished 2024-09-16T08:24:52.794 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:24:52 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.7"} : dispatch 2024-09-16T08:24:52.794 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:24:52 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.7"}]': finished 2024-09-16T08:24:52.794 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:24:52 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "osd require-osd-release", "release": "squid"} : dispatch 2024-09-16T08:24:53.082 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:24:52 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:24:53.082 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:24:52 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:24:53.082 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:24:52 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-16T08:24:53.082 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:24:52 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-16T08:24:53.082 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:24:52 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:24:53.082 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:24:52 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:24:53.083 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:24:52 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-16T08:24:53.083 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:24:52 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "versions"} : dispatch 2024-09-16T08:24:53.083 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:24:52 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "versions"} : dispatch 2024-09-16T08:24:53.083 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:24:52 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "versions"} : dispatch 2024-09-16T08:24:53.083 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:24:52 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "versions"} : dispatch 2024-09-16T08:24:53.083 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:24:52 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:24:53.083 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:24:52 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.0"} : dispatch 2024-09-16T08:24:53.083 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:24:52 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.0"}]': finished 2024-09-16T08:24:53.083 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:24:52 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.1"} : dispatch 2024-09-16T08:24:53.083 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:24:52 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.1"}]': finished 2024-09-16T08:24:53.083 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:24:52 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.2"} : dispatch 2024-09-16T08:24:53.083 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:24:52 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.2"}]': finished 2024-09-16T08:24:53.083 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:24:52 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.3"} : dispatch 2024-09-16T08:24:53.084 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:24:52 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.3"}]': finished 2024-09-16T08:24:53.084 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:24:52 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.4"} : dispatch 2024-09-16T08:24:53.084 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:24:52 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.4"}]': finished 2024-09-16T08:24:53.084 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:24:52 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.5"} : dispatch 2024-09-16T08:24:53.084 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:24:52 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.5"}]': finished 2024-09-16T08:24:53.084 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:24:52 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.6"} : dispatch 2024-09-16T08:24:53.084 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:24:52 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.6"}]': finished 2024-09-16T08:24:53.084 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:24:52 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.7"} : dispatch 2024-09-16T08:24:53.084 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:24:52 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.7"}]': finished 2024-09-16T08:24:53.085 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:24:52 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "osd require-osd-release", "release": "squid"} : dispatch 2024-09-16T08:24:53.090 INFO:teuthology.orchestra.run.smithi028.stdout: 5 271587 9.24 MB/sec execute 591 sec latency 54.547 ms 2024-09-16T08:24:53.798 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:24:53 smithi028 ceph-mon[64115]: Upgrade: Setting container_image for all osd 2024-09-16T08:24:53.798 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:24:53 smithi028 ceph-mon[64115]: Upgrade: Setting require_osd_release to 19 squid 2024-09-16T08:24:53.798 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:24:53 smithi028 ceph-mon[64115]: pgmap v291: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 346 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 8.0 KiB/s rd, 4.8 MiB/s wr, 513 op/s; 251/1656 objects degraded (15.157%) 2024-09-16T08:24:53.798 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:24:53 smithi028 ceph-mon[64115]: Health check cleared: OSD_UPGRADE_FINISHED (was: all OSDs are running squid or later but require_osd_release < squid) 2024-09-16T08:24:53.798 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:24:53 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd='[{"prefix": "osd require-osd-release", "release": "squid"}]': finished 2024-09-16T08:24:53.798 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:24:53 smithi028 ceph-mon[64115]: osdmap e148: 8 total, 7 up, 8 in 2024-09-16T08:24:53.798 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:24:53 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "mds ok-to-stop", "ids": ["foofs.smithi028.srtgev"]} : dispatch 2024-09-16T08:24:54.082 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:24:53 smithi144 ceph-mon[49149]: Upgrade: Setting container_image for all osd 2024-09-16T08:24:54.082 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:24:53 smithi144 ceph-mon[49149]: Upgrade: Setting require_osd_release to 19 squid 2024-09-16T08:24:54.082 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:24:53 smithi144 ceph-mon[49149]: pgmap v291: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 346 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 8.0 KiB/s rd, 4.8 MiB/s wr, 513 op/s; 251/1656 objects degraded (15.157%) 2024-09-16T08:24:54.082 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:24:53 smithi144 ceph-mon[49149]: Health check cleared: OSD_UPGRADE_FINISHED (was: all OSDs are running squid or later but require_osd_release < squid) 2024-09-16T08:24:54.082 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:24:53 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd='[{"prefix": "osd require-osd-release", "release": "squid"}]': finished 2024-09-16T08:24:54.082 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:24:53 smithi144 ceph-mon[49149]: osdmap e148: 8 total, 7 up, 8 in 2024-09-16T08:24:54.082 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:24:53 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "mds ok-to-stop", "ids": ["foofs.smithi028.srtgev"]} : dispatch 2024-09-16T08:24:54.090 INFO:teuthology.orchestra.run.smithi028.stdout: 5 271886 9.25 MB/sec execute 592 sec latency 62.403 ms 2024-09-16T08:24:54.802 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:24:54 smithi028 ceph-mon[64115]: Upgrade: It appears safe to stop mds.foofs.smithi028.srtgev 2024-09-16T08:24:55.081 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:24:54 smithi144 ceph-mon[49149]: Upgrade: It appears safe to stop mds.foofs.smithi028.srtgev 2024-09-16T08:24:55.090 INFO:teuthology.orchestra.run.smithi028.stdout: 5 272304 9.25 MB/sec execute 593 sec latency 42.928 ms 2024-09-16T08:24:55.785 INFO:journalctl@ceph.osd.7.smithi144.stdout:Sep 16 08:24:55 smithi144 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-7[59146]: 2024-09-16T08:24:55.396+0000 7f4aa14a0640 -1 osd.7 0 read_superblock omap replica is missing. 2024-09-16T08:24:55.813 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:24:55 smithi028 ceph-mon[64115]: pgmap v293: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 348 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 7.4 KiB/s rd, 4.3 MiB/s wr, 414 op/s; 258/1680 objects degraded (15.357%) 2024-09-16T08:24:55.813 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:24:55 smithi028 ceph-mon[64115]: Health check update: Degraded data redundancy: 251/1656 objects degraded (15.157%), 35 pgs degraded (PG_DEGRADED) 2024-09-16T08:24:56.081 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:24:55 smithi144 ceph-mon[49149]: pgmap v293: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 348 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 7.4 KiB/s rd, 4.3 MiB/s wr, 414 op/s; 258/1680 objects degraded (15.357%) 2024-09-16T08:24:56.082 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:24:55 smithi144 ceph-mon[49149]: Health check update: Degraded data redundancy: 251/1656 objects degraded (15.157%), 35 pgs degraded (PG_DEGRADED) 2024-09-16T08:24:56.090 INFO:teuthology.orchestra.run.smithi028.stdout: 5 272593 9.24 MB/sec execute 594 sec latency 68.028 ms 2024-09-16T08:24:57.074 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:24:56 smithi028 ceph-mon[64115]: Upgrade: Updating mds.foofs.smithi028.srtgev 2024-09-16T08:24:57.074 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:24:56 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:24:57.075 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:24:56 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "auth get-or-create", "entity": "mds.foofs.smithi028.srtgev", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-09-16T08:24:57.075 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:24:56 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-16T08:24:57.075 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:24:56 smithi028 ceph-mon[64115]: Deploying daemon mds.foofs.smithi028.srtgev on smithi028 2024-09-16T08:24:57.075 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:24:56 smithi028 ceph-mon[64115]: from='osd.7 [v2:172.21.15.144:6824/3651631052,v1:172.21.15.144:6825/3651631052]' entity='osd.7' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]} : dispatch 2024-09-16T08:24:57.075 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:24:56 smithi028 ceph-mon[64115]: from='osd.7 ' entity='osd.7' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]} : dispatch 2024-09-16T08:24:57.091 INFO:teuthology.orchestra.run.smithi028.stdout: 5 272837 9.23 MB/sec execute 595 sec latency 44.013 ms 2024-09-16T08:24:57.332 INFO:journalctl@ceph.osd.7.smithi144.stdout:Sep 16 08:24:56 smithi144 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-7[59146]: 2024-09-16T08:24:56.925+0000 7f4aa14a0640 -1 osd.7 145 log_to_monitors true 2024-09-16T08:24:57.332 INFO:journalctl@ceph.osd.7.smithi144.stdout:Sep 16 08:24:56 smithi144 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-7[59146]: 2024-09-16T08:24:56.959+0000 7f4a9924a640 -1 osd.7 145 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-16T08:24:57.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:24:56 smithi144 ceph-mon[49149]: Upgrade: Updating mds.foofs.smithi028.srtgev 2024-09-16T08:24:57.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:24:56 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:24:57.333 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:24:56 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "auth get-or-create", "entity": "mds.foofs.smithi028.srtgev", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-09-16T08:24:57.333 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:24:56 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-16T08:24:57.333 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:24:56 smithi144 ceph-mon[49149]: Deploying daemon mds.foofs.smithi028.srtgev on smithi028 2024-09-16T08:24:57.333 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:24:56 smithi144 ceph-mon[49149]: from='osd.7 [v2:172.21.15.144:6824/3651631052,v1:172.21.15.144:6825/3651631052]' entity='osd.7' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]} : dispatch 2024-09-16T08:24:57.333 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:24:56 smithi144 ceph-mon[49149]: from='osd.7 ' entity='osd.7' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]} : dispatch 2024-09-16T08:24:58.091 INFO:teuthology.orchestra.run.smithi028.stdout: 5 273062 9.23 MB/sec execute 596 sec latency 106.971 ms 2024-09-16T08:24:58.106 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:24:57 smithi028 ceph-mon[64115]: pgmap v294: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 346 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 6.3 KiB/s rd, 4.4 MiB/s wr, 439 op/s; 259/1734 objects degraded (14.937%) 2024-09-16T08:24:58.106 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:24:57 smithi028 ceph-mon[64115]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-09-16T08:24:58.106 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:24:57 smithi028 ceph-mon[64115]: osdmap e149: 8 total, 7 up, 8 in 2024-09-16T08:24:58.106 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:24:57 smithi028 ceph-mon[64115]: from='osd.7 [v2:172.21.15.144:6824/3651631052,v1:172.21.15.144:6825/3651631052]' entity='osd.7' cmd={"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi144", "root=default"]} : dispatch 2024-09-16T08:24:58.106 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:24:57 smithi028 ceph-mon[64115]: from='osd.7 ' entity='osd.7' cmd={"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi144", "root=default"]} : dispatch 2024-09-16T08:24:58.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:24:57 smithi144 ceph-mon[49149]: pgmap v294: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 346 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 6.3 KiB/s rd, 4.4 MiB/s wr, 439 op/s; 259/1734 objects degraded (14.937%) 2024-09-16T08:24:58.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:24:57 smithi144 ceph-mon[49149]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-09-16T08:24:58.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:24:57 smithi144 ceph-mon[49149]: osdmap e149: 8 total, 7 up, 8 in 2024-09-16T08:24:58.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:24:57 smithi144 ceph-mon[49149]: from='osd.7 [v2:172.21.15.144:6824/3651631052,v1:172.21.15.144:6825/3651631052]' entity='osd.7' cmd={"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi144", "root=default"]} : dispatch 2024-09-16T08:24:58.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:24:57 smithi144 ceph-mon[49149]: from='osd.7 ' entity='osd.7' cmd={"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi144", "root=default"]} : dispatch 2024-09-16T08:24:59.091 INFO:teuthology.orchestra.run.smithi028.stdout: 5 273171 9.22 MB/sec execute 597 sec latency 1107.003 ms 2024-09-16T08:24:59.332 INFO:journalctl@ceph.osd.5.smithi144.stdout:Sep 16 08:24:59 smithi144 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-5[55067]: 2024-09-16T08:24:59.226+0000 7fdc4dd8a640 -1 bdev(0x559e0c666c00 /var/lib/ceph/osd/ceph-5/block) aio_submit retries 5 2024-09-16T08:24:59.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:24:58 smithi144 ceph-mon[49149]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-16T08:24:59.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:24:58 smithi144 ceph-mon[49149]: osd.7 [v2:172.21.15.144:6824/3651631052,v1:172.21.15.144:6825/3651631052] boot 2024-09-16T08:24:59.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:24:58 smithi144 ceph-mon[49149]: osdmap e150: 8 total, 8 up, 8 in 2024-09-16T08:24:59.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:24:58 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "osd metadata", "id": 7} : dispatch 2024-09-16T08:24:59.333 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:24:58 smithi144 ceph-mon[49149]: osdmap e151: 8 total, 8 up, 8 in 2024-09-16T08:24:59.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:24:58 smithi028 ceph-mon[64115]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-16T08:24:59.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:24:58 smithi028 ceph-mon[64115]: osd.7 [v2:172.21.15.144:6824/3651631052,v1:172.21.15.144:6825/3651631052] boot 2024-09-16T08:24:59.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:24:58 smithi028 ceph-mon[64115]: osdmap e150: 8 total, 8 up, 8 in 2024-09-16T08:24:59.448 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:24:58 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "osd metadata", "id": 7} : dispatch 2024-09-16T08:24:59.448 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:24:58 smithi028 ceph-mon[64115]: osdmap e151: 8 total, 8 up, 8 in 2024-09-16T08:24:59.448 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:24:59 smithi028 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-mon-smithi028[64111]: 2024-09-16T08:24:59.098+0000 7f332caa2640 -1 log_channel(cluster) log [ERR] : Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-09-16T08:25:00.091 INFO:teuthology.orchestra.run.smithi028.stdout: 5 273173 9.20 MB/sec execute 598 sec latency 1183.965 ms 2024-09-16T08:25:00.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:24:59 smithi144 ceph-mon[49149]: pgmap v297: 97 pgs: 3 peering, 7 active+undersized, 32 active+undersized+degraded, 55 active+clean; 350 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 4.8 KiB/s rd, 4.8 MiB/s wr, 408 op/s; 230/1752 objects degraded (13.128%) 2024-09-16T08:25:00.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:24:59 smithi144 ceph-mon[49149]: Health check failed: 1 filesystem is degraded (FS_DEGRADED) 2024-09-16T08:25:00.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:24:59 smithi144 ceph-mon[49149]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-09-16T08:25:00.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:24:59 smithi144 ceph-mon[49149]: osdmap e152: 8 total, 8 up, 8 in 2024-09-16T08:25:00.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:24:59 smithi144 ceph-mon[49149]: Standby daemon mds.foofs.smithi144.oozgsi assigned to filesystem foofs as rank 0 2024-09-16T08:25:00.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:24:59 smithi144 ceph-mon[49149]: Health check failed: insufficient standby MDS daemons available (MDS_INSUFFICIENT_STANDBY) 2024-09-16T08:25:00.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:24:59 smithi144 ceph-mon[49149]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-09-16T08:25:00.333 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:24:59 smithi144 ceph-mon[49149]: fsmap foofs:0/1 1 up:standby, 1 failed 2024-09-16T08:25:00.333 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:24:59 smithi144 ceph-mon[49149]: fsmap foofs:1/1 {0=foofs.smithi144.oozgsi=up:replay} 2024-09-16T08:25:00.333 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:24:59 smithi144 ceph-mon[49149]: Health check update: Degraded data redundancy: 230/1752 objects degraded (13.128%), 32 pgs degraded (PG_DEGRADED) 2024-09-16T08:25:00.333 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:24:59 smithi144 ceph-mon[49149]: osdmap e153: 8 total, 8 up, 8 in 2024-09-16T08:25:00.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:24:59 smithi028 ceph-mon[64115]: pgmap v297: 97 pgs: 3 peering, 7 active+undersized, 32 active+undersized+degraded, 55 active+clean; 350 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 4.8 KiB/s rd, 4.8 MiB/s wr, 408 op/s; 230/1752 objects degraded (13.128%) 2024-09-16T08:25:00.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:24:59 smithi028 ceph-mon[64115]: Health check failed: 1 filesystem is degraded (FS_DEGRADED) 2024-09-16T08:25:00.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:24:59 smithi028 ceph-mon[64115]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-09-16T08:25:00.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:24:59 smithi028 ceph-mon[64115]: osdmap e152: 8 total, 8 up, 8 in 2024-09-16T08:25:00.448 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:24:59 smithi028 ceph-mon[64115]: Standby daemon mds.foofs.smithi144.oozgsi assigned to filesystem foofs as rank 0 2024-09-16T08:25:00.448 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:24:59 smithi028 ceph-mon[64115]: Health check failed: insufficient standby MDS daemons available (MDS_INSUFFICIENT_STANDBY) 2024-09-16T08:25:00.448 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:24:59 smithi028 ceph-mon[64115]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-09-16T08:25:00.448 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:24:59 smithi028 ceph-mon[64115]: fsmap foofs:0/1 1 up:standby, 1 failed 2024-09-16T08:25:00.448 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:24:59 smithi028 ceph-mon[64115]: fsmap foofs:1/1 {0=foofs.smithi144.oozgsi=up:replay} 2024-09-16T08:25:00.448 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:24:59 smithi028 ceph-mon[64115]: Health check update: Degraded data redundancy: 230/1752 objects degraded (13.128%), 32 pgs degraded (PG_DEGRADED) 2024-09-16T08:25:00.448 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:24:59 smithi028 ceph-mon[64115]: osdmap e153: 8 total, 8 up, 8 in 2024-09-16T08:25:01.091 INFO:teuthology.orchestra.run.smithi028.stdout: 5 273173 9.18 MB/sec execute 599 sec latency 1995.712 ms 2024-09-16T08:25:02.091 INFO:teuthology.orchestra.run.smithi028.stdout: 5 cleanup 600 sec 2024-09-16T08:25:02.197 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:25:01 smithi028 ceph-mon[64115]: pgmap v301: 97 pgs: 2 remapped+peering, 4 peering, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 2 active+undersized, 21 active+undersized+degraded, 66 active+clean; 349 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 4.0 MiB/s rd, 4.1 MiB/s wr, 469 op/s; 127/1674 objects degraded (7.587%); 382 KiB/s, 23 objects/s recovering 2024-09-16T08:25:02.197 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:25:01 smithi028 ceph-mon[64115]: osdmap e154: 8 total, 8 up, 8 in 2024-09-16T08:25:02.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:25:01 smithi144 ceph-mon[49149]: pgmap v301: 97 pgs: 2 remapped+peering, 4 peering, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 2 active+undersized, 21 active+undersized+degraded, 66 active+clean; 349 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 4.0 MiB/s rd, 4.1 MiB/s wr, 469 op/s; 127/1674 objects degraded (7.587%); 382 KiB/s, 23 objects/s recovering 2024-09-16T08:25:02.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:25:01 smithi144 ceph-mon[49149]: osdmap e154: 8 total, 8 up, 8 in 2024-09-16T08:25:03.091 INFO:teuthology.orchestra.run.smithi028.stdout: 5 cleanup 601 sec 2024-09-16T08:25:03.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:25:03 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:25:03.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:25:03 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-16T08:25:03.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:25:03 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "mds metadata", "who": "foofs.smithi028.srtgev"} : dispatch 2024-09-16T08:25:03.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:25:03 smithi028 ceph-mon[64115]: pgmap v303: 97 pgs: 4 peering, 1 active+recovery_wait+degraded, 92 active+clean; 344 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 33 MiB/s rd, 2.3 MiB/s wr, 253 op/s; 7/1653 objects degraded (0.423%); 2.6 MiB/s, 56 objects/s recovering 2024-09-16T08:25:03.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:25:03 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:25:03.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:25:03 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-16T08:25:03.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:25:03 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "mds metadata", "who": "foofs.smithi028.srtgev"} : dispatch 2024-09-16T08:25:03.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:25:03 smithi144 ceph-mon[49149]: pgmap v303: 97 pgs: 4 peering, 1 active+recovery_wait+degraded, 92 active+clean; 344 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 33 MiB/s rd, 2.3 MiB/s wr, 253 op/s; 7/1653 objects degraded (0.423%); 2.6 MiB/s, 56 objects/s recovering 2024-09-16T08:25:04.091 INFO:teuthology.orchestra.run.smithi028.stdout: 5 cleanup 602 sec 2024-09-16T08:25:04.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:25:04 smithi144 ceph-mon[49149]: Health check cleared: MDS_INSUFFICIENT_STANDBY (was: insufficient standby MDS daemons available) 2024-09-16T08:25:04.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:25:04 smithi144 ceph-mon[49149]: mds.? [v2:172.21.15.28:6834/2040229326,v1:172.21.15.28:6835/2040229326] up:boot 2024-09-16T08:25:04.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:25:04 smithi144 ceph-mon[49149]: fsmap foofs:1/1 {0=foofs.smithi144.oozgsi=up:replay} 1 up:standby 2024-09-16T08:25:04.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:25:04 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "mds metadata", "who": "foofs.smithi028.srtgev"} : dispatch 2024-09-16T08:25:04.697 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:25:04 smithi028 ceph-mon[64115]: Health check cleared: MDS_INSUFFICIENT_STANDBY (was: insufficient standby MDS daemons available) 2024-09-16T08:25:04.697 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:25:04 smithi028 ceph-mon[64115]: mds.? [v2:172.21.15.28:6834/2040229326,v1:172.21.15.28:6835/2040229326] up:boot 2024-09-16T08:25:04.697 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:25:04 smithi028 ceph-mon[64115]: fsmap foofs:1/1 {0=foofs.smithi144.oozgsi=up:replay} 1 up:standby 2024-09-16T08:25:04.698 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:25:04 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "mds metadata", "who": "foofs.smithi028.srtgev"} : dispatch 2024-09-16T08:25:05.091 INFO:teuthology.orchestra.run.smithi028.stdout: 5 cleanup 603 sec 2024-09-16T08:25:05.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:25:05 smithi144 ceph-mon[49149]: reconnect by client.24359 172.21.15.144:0/2699919051 after 0.000999983 2024-09-16T08:25:05.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:25:05 smithi144 ceph-mon[49149]: mds.? [v2:172.21.15.144:6832/2267142930,v1:172.21.15.144:6833/2267142930] up:reconnect 2024-09-16T08:25:05.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:25:05 smithi144 ceph-mon[49149]: fsmap foofs:1/1 {0=foofs.smithi144.oozgsi=up:reconnect} 1 up:standby 2024-09-16T08:25:05.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:25:05 smithi144 ceph-mon[49149]: reconnect by client.14546 172.21.15.28:0/1295173080 after 0.0569991 2024-09-16T08:25:05.583 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:25:05 smithi144 ceph-mon[49149]: pgmap v304: 97 pgs: 1 peering, 96 active+clean; 344 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 31 MiB/s rd, 1.5 MiB/s wr, 172 op/s; 2.4 MiB/s, 46 objects/s recovering 2024-09-16T08:25:05.583 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:25:05 smithi144 ceph-mon[49149]: Health check update: Degraded data redundancy: 7/1653 objects degraded (0.423%), 1 pg degraded (PG_DEGRADED) 2024-09-16T08:25:05.697 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:25:05 smithi028 ceph-mon[64115]: reconnect by client.24359 172.21.15.144:0/2699919051 after 0.000999983 2024-09-16T08:25:05.697 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:25:05 smithi028 ceph-mon[64115]: mds.? [v2:172.21.15.144:6832/2267142930,v1:172.21.15.144:6833/2267142930] up:reconnect 2024-09-16T08:25:05.697 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:25:05 smithi028 ceph-mon[64115]: fsmap foofs:1/1 {0=foofs.smithi144.oozgsi=up:reconnect} 1 up:standby 2024-09-16T08:25:05.697 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:25:05 smithi028 ceph-mon[64115]: reconnect by client.14546 172.21.15.28:0/1295173080 after 0.0569991 2024-09-16T08:25:05.697 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:25:05 smithi028 ceph-mon[64115]: pgmap v304: 97 pgs: 1 peering, 96 active+clean; 344 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 31 MiB/s rd, 1.5 MiB/s wr, 172 op/s; 2.4 MiB/s, 46 objects/s recovering 2024-09-16T08:25:05.697 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:25:05 smithi028 ceph-mon[64115]: Health check update: Degraded data redundancy: 7/1653 objects degraded (0.423%), 1 pg degraded (PG_DEGRADED) 2024-09-16T08:25:06.091 INFO:teuthology.orchestra.run.smithi028.stdout: 5 cleanup 604 sec 2024-09-16T08:25:06.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:25:06 smithi144 ceph-mon[49149]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 7/1653 objects degraded (0.423%), 1 pg degraded) 2024-09-16T08:25:06.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:25:06 smithi144 ceph-mon[49149]: mds.? [v2:172.21.15.144:6832/2267142930,v1:172.21.15.144:6833/2267142930] up:rejoin 2024-09-16T08:25:06.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:25:06 smithi144 ceph-mon[49149]: fsmap foofs:1/1 {0=foofs.smithi144.oozgsi=up:rejoin} 1 up:standby 2024-09-16T08:25:06.697 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:25:06 smithi028 ceph-mon[64115]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 7/1653 objects degraded (0.423%), 1 pg degraded) 2024-09-16T08:25:06.698 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:25:06 smithi028 ceph-mon[64115]: mds.? [v2:172.21.15.144:6832/2267142930,v1:172.21.15.144:6833/2267142930] up:rejoin 2024-09-16T08:25:06.698 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:25:06 smithi028 ceph-mon[64115]: fsmap foofs:1/1 {0=foofs.smithi144.oozgsi=up:rejoin} 1 up:standby 2024-09-16T08:25:07.091 INFO:teuthology.orchestra.run.smithi028.stdout: 5 cleanup 605 sec 2024-09-16T08:25:07.560 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:25:07 smithi028 ceph-mon[64115]: mds.? [v2:172.21.15.144:6832/2267142930,v1:172.21.15.144:6833/2267142930] up:clientreplay 2024-09-16T08:25:07.560 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:25:07 smithi028 ceph-mon[64115]: fsmap foofs:1/1 {0=foofs.smithi144.oozgsi=up:clientreplay} 1 up:standby 2024-09-16T08:25:07.560 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:25:07 smithi028 ceph-mon[64115]: daemon mds.foofs.smithi144.oozgsi is now active in filesystem foofs as rank 0 2024-09-16T08:25:07.560 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:25:07 smithi028 ceph-mon[64115]: pgmap v305: 97 pgs: 97 active+clean; 344 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 26 MiB/s rd, 1.1 MiB/s wr, 134 op/s; 1.9 MiB/s, 35 objects/s recovering 2024-09-16T08:25:07.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:25:07 smithi144 ceph-mon[49149]: mds.? [v2:172.21.15.144:6832/2267142930,v1:172.21.15.144:6833/2267142930] up:clientreplay 2024-09-16T08:25:07.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:25:07 smithi144 ceph-mon[49149]: fsmap foofs:1/1 {0=foofs.smithi144.oozgsi=up:clientreplay} 1 up:standby 2024-09-16T08:25:07.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:25:07 smithi144 ceph-mon[49149]: daemon mds.foofs.smithi144.oozgsi is now active in filesystem foofs as rank 0 2024-09-16T08:25:07.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:25:07 smithi144 ceph-mon[49149]: pgmap v305: 97 pgs: 97 active+clean; 344 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 26 MiB/s rd, 1.1 MiB/s wr, 134 op/s; 1.9 MiB/s, 35 objects/s recovering 2024-09-16T08:25:08.091 INFO:teuthology.orchestra.run.smithi028.stdout: 5 cleanup 606 sec 2024-09-16T08:25:08.329 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:25:08 smithi028 ceph-mon[64115]: Health check cleared: FS_DEGRADED (was: 1 filesystem is degraded) 2024-09-16T08:25:08.329 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:25:08 smithi028 ceph-mon[64115]: Cluster is now healthy 2024-09-16T08:25:08.329 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:25:08 smithi028 ceph-mon[64115]: mds.? [v2:172.21.15.144:6832/2267142930,v1:172.21.15.144:6833/2267142930] up:active 2024-09-16T08:25:08.329 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:25:08 smithi028 ceph-mon[64115]: fsmap foofs:1 {0=foofs.smithi144.oozgsi=up:active} 1 up:standby 2024-09-16T08:25:08.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:25:08 smithi144 ceph-mon[49149]: Health check cleared: FS_DEGRADED (was: 1 filesystem is degraded) 2024-09-16T08:25:08.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:25:08 smithi144 ceph-mon[49149]: Cluster is now healthy 2024-09-16T08:25:08.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:25:08 smithi144 ceph-mon[49149]: mds.? [v2:172.21.15.144:6832/2267142930,v1:172.21.15.144:6833/2267142930] up:active 2024-09-16T08:25:08.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:25:08 smithi144 ceph-mon[49149]: fsmap foofs:1 {0=foofs.smithi144.oozgsi=up:active} 1 up:standby 2024-09-16T08:25:09.091 INFO:teuthology.orchestra.run.smithi028.stdout: 1 cleanup 607 sec 2024-09-16T08:25:09.160 INFO:teuthology.orchestra.run.smithi028.stdout: 0 cleanup 607 sec 2024-09-16T08:25:09.160 INFO:teuthology.orchestra.run.smithi028.stdout: 2024-09-16T08:25:09.160 INFO:teuthology.orchestra.run.smithi028.stdout: Operation Count AvgLat MaxLat 2024-09-16T08:25:09.161 INFO:teuthology.orchestra.run.smithi028.stdout: ---------------------------------------- 2024-09-16T08:25:09.161 INFO:teuthology.orchestra.run.smithi028.stdout: NTCreateX 176456 4.908 8293.037 2024-09-16T08:25:09.161 INFO:teuthology.orchestra.run.smithi028.stdout: Close 129541 0.145 10.996 2024-09-16T08:25:09.161 INFO:teuthology.orchestra.run.smithi028.stdout: Rename 7483 10.529 1078.687 2024-09-16T08:25:09.161 INFO:teuthology.orchestra.run.smithi028.stdout: Unlink 35694 7.147 2144.719 2024-09-16T08:25:09.161 INFO:teuthology.orchestra.run.smithi028.stdout: Qpathinfo 160035 1.313 8229.668 2024-09-16T08:25:09.161 INFO:teuthology.orchestra.run.smithi028.stdout: Qfileinfo 27902 0.007 0.625 2024-09-16T08:25:09.161 INFO:teuthology.orchestra.run.smithi028.stdout: Qfsinfo 29356 0.682 41.724 2024-09-16T08:25:09.161 INFO:teuthology.orchestra.run.smithi028.stdout: Sfileinfo 14386 13.926 8219.268 2024-09-16T08:25:09.161 INFO:teuthology.orchestra.run.smithi028.stdout: Find 61859 1.056 31.028 2024-09-16T08:25:09.162 INFO:teuthology.orchestra.run.smithi028.stdout: WriteX 87193 14.963 8220.302 2024-09-16T08:25:09.162 INFO:teuthology.orchestra.run.smithi028.stdout: ReadX 276954 0.011 3.341 2024-09-16T08:25:09.162 INFO:teuthology.orchestra.run.smithi028.stdout: LockX 576 0.406 1.773 2024-09-16T08:25:09.162 INFO:teuthology.orchestra.run.smithi028.stdout: UnlockX 576 0.371 4.088 2024-09-16T08:25:09.162 INFO:teuthology.orchestra.run.smithi028.stdout: Flush 12357 0.003 0.044 2024-09-16T08:25:09.162 INFO:teuthology.orchestra.run.smithi028.stdout: 2024-09-16T08:25:09.162 INFO:teuthology.orchestra.run.smithi028.stdout:Throughput 9.18484 MB/sec 5 clients 5 procs max_latency=2385.335 ms 2024-09-16T08:25:09.164 DEBUG:teuthology.orchestra.run.smithi028:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'umount /mnt/foo' 2024-09-16T08:25:09.216 DEBUG:teuthology.orchestra.run.smithi028:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'while ! mount -t nfs $(hostname):/fake /mnt/foo ; do sleep 5 ; done' 2024-09-16T08:25:09.303 DEBUG:teuthology.orchestra.run.smithi028:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'cd /mnt/foo && dbench 5 -t 5' 2024-09-16T08:25:09.329 INFO:teuthology.orchestra.run.smithi028.stdout:dbench version 4.00 - Copyright Andrew Tridgell 1999-2004 2024-09-16T08:25:09.330 INFO:teuthology.orchestra.run.smithi028.stdout: 2024-09-16T08:25:09.330 INFO:teuthology.orchestra.run.smithi028.stdout:Running for 5 seconds with load '/usr/share/dbench/client.txt' and minimum warmup 1 secs 2024-09-16T08:25:09.330 INFO:teuthology.orchestra.run.smithi028.stdout:3 of 5 processes prepared for launch 0 sec 2024-09-16T08:25:09.430 INFO:teuthology.orchestra.run.smithi028.stdout:5 of 5 processes prepared for launch 0 sec 2024-09-16T08:25:09.431 INFO:teuthology.orchestra.run.smithi028.stdout:releasing clients 2024-09-16T08:25:09.432 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:25:09 smithi028 ceph-mon[64115]: pgmap v306: 97 pgs: 97 active+clean; 340 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 23 MiB/s rd, 466 KiB/s wr, 51 op/s; 1.6 MiB/s, 24 objects/s recovering 2024-09-16T08:25:09.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:25:09 smithi144 ceph-mon[49149]: pgmap v306: 97 pgs: 97 active+clean; 340 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 23 MiB/s rd, 466 KiB/s wr, 51 op/s; 1.6 MiB/s, 24 objects/s recovering 2024-09-16T08:25:11.431 INFO:teuthology.orchestra.run.smithi028.stdout: 5 379 25.23 MB/sec execute 1 sec latency 56.901 ms 2024-09-16T08:25:11.807 INFO:teuthology.orchestra.run.smithi028.stdout:true 2024-09-16T08:25:12.027 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:25:11 smithi028 ceph-mon[64115]: pgmap v307: 97 pgs: 97 active+clean; 342 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 21 MiB/s rd, 1.6 MiB/s wr, 80 op/s; 1.3 MiB/s, 18 objects/s recovering 2024-09-16T08:25:12.058 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:25:11 smithi028 ceph-mon[64115]: daemon mds.foofs.smithi028.srtgev restarted 2024-09-16T08:25:12.058 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:25:11 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:25:12.058 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:25:11 smithi028 ceph-mon[64115]: osdmap e155: 8 total, 8 up, 8 in 2024-09-16T08:25:12.058 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:25:11 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:25:12.058 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:25:11 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-16T08:25:12.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:25:11 smithi144 ceph-mon[49149]: pgmap v307: 97 pgs: 97 active+clean; 342 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 21 MiB/s rd, 1.6 MiB/s wr, 80 op/s; 1.3 MiB/s, 18 objects/s recovering 2024-09-16T08:25:12.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:25:11 smithi144 ceph-mon[49149]: daemon mds.foofs.smithi028.srtgev restarted 2024-09-16T08:25:12.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:25:11 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:25:12.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:25:11 smithi144 ceph-mon[49149]: osdmap e155: 8 total, 8 up, 8 in 2024-09-16T08:25:12.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:25:11 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:25:12.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:25:11 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-16T08:25:12.433 INFO:teuthology.orchestra.run.smithi028.stdout: 5 510 23.05 MB/sec execute 2 sec latency 38.151 ms 2024-09-16T08:25:13.002 INFO:teuthology.orchestra.run.smithi028.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-16T08:25:13.002 INFO:teuthology.orchestra.run.smithi028.stdout:alertmanager.smithi028 smithi028 *:9093,9094 running (9m) 3m ago 18m 18.6M - 0.27.0 11f11916f8cd a14fcb4cb2b6 2024-09-16T08:25:13.002 INFO:teuthology.orchestra.run.smithi028.stdout:crash.smithi028 smithi028 running (6m) 3m ago 18m 6899k - 19.3.0-4917-g8293d73f 425e316ba8eb 4bb89df2a867 2024-09-16T08:25:13.002 INFO:teuthology.orchestra.run.smithi028.stdout:crash.smithi144 smithi144 running (6m) 25s ago 18m 6895k - 19.3.0-4917-g8293d73f 425e316ba8eb 24a96875d7af 2024-09-16T08:25:13.002 INFO:teuthology.orchestra.run.smithi028.stdout:grafana.smithi028 smithi028 *:3000 running (9m) 3m ago 18m 45.9M - dad864ee21e9 b55ba8b944a1 2024-09-16T08:25:13.002 INFO:teuthology.orchestra.run.smithi028.stdout:mds.foofs.smithi028.srtgev smithi028 starting - - - - 2024-09-16T08:25:13.002 INFO:teuthology.orchestra.run.smithi028.stdout:mds.foofs.smithi144.oozgsi smithi144 running (12m) 25s ago 12m 17.7M - 17.2.0 e1d6a67b021e ecf4c2bb0fdc 2024-09-16T08:25:13.003 INFO:teuthology.orchestra.run.smithi028.stdout:mgr.smithi028.azvyzy smithi028 *:8443,9283,8765 running (8m) 3m ago 19m 519M - 19.3.0-4917-g8293d73f 425e316ba8eb e01d4382d03f 2024-09-16T08:25:13.003 INFO:teuthology.orchestra.run.smithi028.stdout:mgr.smithi144.utzszj smithi144 *:8443,9283,8765 running (7m) 25s ago 17m 458M - 19.3.0-4917-g8293d73f 425e316ba8eb 71f7f802bffb 2024-09-16T08:25:13.003 INFO:teuthology.orchestra.run.smithi028.stdout:mon.smithi028 smithi028 running (7m) 3m ago 19m 49.8M 2048M 19.3.0-4917-g8293d73f 425e316ba8eb 14efcedc0088 2024-09-16T08:25:13.003 INFO:teuthology.orchestra.run.smithi028.stdout:mon.smithi144 smithi144 running (7m) 25s ago 17m 46.9M 2048M 19.3.0-4917-g8293d73f 425e316ba8eb c081c5ed44b5 2024-09-16T08:25:13.003 INFO:teuthology.orchestra.run.smithi028.stdout:nfs.foo.0.0.smithi028.dkttgj smithi028 *:2049 running (12m) 3m ago 12m 285M - 4.0 e1d6a67b021e b5dccc45a6ed 2024-09-16T08:25:13.003 INFO:teuthology.orchestra.run.smithi028.stdout:nfs.foo.1.0.smithi144.epipkg smithi144 *:2049 running (12m) 25s ago 12m 87.5M - 4.0 e1d6a67b021e faa89e61f50e 2024-09-16T08:25:13.003 INFO:teuthology.orchestra.run.smithi028.stdout:node-exporter.smithi028 smithi028 *:9100 running (9m) 3m ago 18m 9.92M - 1.7.0 72c9c2088986 9e2b8e8a63a9 2024-09-16T08:25:13.003 INFO:teuthology.orchestra.run.smithi028.stdout:node-exporter.smithi144 smithi144 *:9100 running (8m) 25s ago 17m 10.1M - 1.7.0 72c9c2088986 78834c5e992f 2024-09-16T08:25:13.003 INFO:teuthology.orchestra.run.smithi028.stdout:osd.0 smithi028 running (5m) 3m ago 17m 354M 1937M 19.3.0-4917-g8293d73f 425e316ba8eb 2fc02ba4f14f 2024-09-16T08:25:13.003 INFO:teuthology.orchestra.run.smithi028.stdout:osd.1 smithi028 running (4m) 3m ago 16m 266M 1937M 19.3.0-4917-g8293d73f 425e316ba8eb df34bd603d8f 2024-09-16T08:25:13.003 INFO:teuthology.orchestra.run.smithi028.stdout:osd.2 smithi028 running (3m) 3m ago 16m 220M 1937M 19.3.0-4917-g8293d73f 425e316ba8eb c92be6327b12 2024-09-16T08:25:13.003 INFO:teuthology.orchestra.run.smithi028.stdout:osd.3 smithi028 running (3m) 3m ago 15m 11.2M 1937M 19.3.0-4917-g8293d73f 425e316ba8eb bd84ba1224f2 2024-09-16T08:25:13.003 INFO:teuthology.orchestra.run.smithi028.stdout:osd.4 smithi144 running (2m) 25s ago 15m 320M 2704M 19.3.0-4917-g8293d73f 425e316ba8eb c48cb68e901e 2024-09-16T08:25:13.003 INFO:teuthology.orchestra.run.smithi028.stdout:osd.5 smithi144 running (106s) 25s ago 15m 311M 2704M 19.3.0-4917-g8293d73f 425e316ba8eb 59ec97fb3cf3 2024-09-16T08:25:13.004 INFO:teuthology.orchestra.run.smithi028.stdout:osd.6 smithi144 running (68s) 25s ago 14m 260M 2704M 19.3.0-4917-g8293d73f 425e316ba8eb b301bf0731f0 2024-09-16T08:25:13.004 INFO:teuthology.orchestra.run.smithi028.stdout:osd.7 smithi144 running (29s) 25s ago 14m 11.4M 2704M 19.3.0-4917-g8293d73f 425e316ba8eb f1a5cbf3a7bd 2024-09-16T08:25:13.004 INFO:teuthology.orchestra.run.smithi028.stdout:prometheus.smithi028 smithi028 *:9095 running (7m) 3m ago 17m 55.3M - 2.51.0 1d3b7f56885b 42667809df2b 2024-09-16T08:25:13.089 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:25:12 smithi028 ceph-mon[64115]: from='client.34460 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T08:25:13.089 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:25:12 smithi028 ceph-mon[64115]: mds.? [v2:172.21.15.28:6834/1804746814,v1:172.21.15.28:6835/1804746814] up:boot 2024-09-16T08:25:13.089 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:25:12 smithi028 ceph-mon[64115]: fsmap foofs:1 {0=foofs.smithi144.oozgsi=up:active} 1 up:standby 2024-09-16T08:25:13.089 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:25:12 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "mds metadata", "who": "foofs.smithi028.srtgev"} : dispatch 2024-09-16T08:25:13.089 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:25:12 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:25:13.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:25:12 smithi144 ceph-mon[49149]: from='client.34460 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T08:25:13.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:25:12 smithi144 ceph-mon[49149]: mds.? [v2:172.21.15.28:6834/1804746814,v1:172.21.15.28:6835/1804746814] up:boot 2024-09-16T08:25:13.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:25:12 smithi144 ceph-mon[49149]: fsmap foofs:1 {0=foofs.smithi144.oozgsi=up:active} 1 up:standby 2024-09-16T08:25:13.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:25:12 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "mds metadata", "who": "foofs.smithi028.srtgev"} : dispatch 2024-09-16T08:25:13.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:25:12 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:25:13.431 INFO:teuthology.orchestra.run.smithi028.stdout: 5 745 21.44 MB/sec execute 3 sec latency 37.215 ms 2024-09-16T08:25:13.663 INFO:teuthology.orchestra.run.smithi028.stdout:{ 2024-09-16T08:25:13.663 INFO:teuthology.orchestra.run.smithi028.stdout: "mon": { 2024-09-16T08:25:13.664 INFO:teuthology.orchestra.run.smithi028.stdout: "ceph version 19.3.0-4917-g8293d73f (8293d73f8690540e843a81caec373f9cc29cf705) squid (dev)": 2 2024-09-16T08:25:13.664 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-09-16T08:25:13.664 INFO:teuthology.orchestra.run.smithi028.stdout: "mgr": { 2024-09-16T08:25:13.664 INFO:teuthology.orchestra.run.smithi028.stdout: "ceph version 19.3.0-4917-g8293d73f (8293d73f8690540e843a81caec373f9cc29cf705) squid (dev)": 2 2024-09-16T08:25:13.664 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-09-16T08:25:13.664 INFO:teuthology.orchestra.run.smithi028.stdout: "osd": { 2024-09-16T08:25:13.664 INFO:teuthology.orchestra.run.smithi028.stdout: "ceph version 19.3.0-4917-g8293d73f (8293d73f8690540e843a81caec373f9cc29cf705) squid (dev)": 8 2024-09-16T08:25:13.664 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-09-16T08:25:13.664 INFO:teuthology.orchestra.run.smithi028.stdout: "mds": { 2024-09-16T08:25:13.664 INFO:teuthology.orchestra.run.smithi028.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 1, 2024-09-16T08:25:13.664 INFO:teuthology.orchestra.run.smithi028.stdout: "ceph version 19.3.0-4917-g8293d73f (8293d73f8690540e843a81caec373f9cc29cf705) squid (dev)": 1 2024-09-16T08:25:13.664 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-09-16T08:25:13.664 INFO:teuthology.orchestra.run.smithi028.stdout: "overall": { 2024-09-16T08:25:13.664 INFO:teuthology.orchestra.run.smithi028.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 1, 2024-09-16T08:25:13.665 INFO:teuthology.orchestra.run.smithi028.stdout: "ceph version 19.3.0-4917-g8293d73f (8293d73f8690540e843a81caec373f9cc29cf705) squid (dev)": 13 2024-09-16T08:25:13.665 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-09-16T08:25:13.665 INFO:teuthology.orchestra.run.smithi028.stdout:} 2024-09-16T08:25:14.106 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:25:13 smithi028 ceph-mon[64115]: from='client.44409 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T08:25:14.106 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:25:13 smithi028 ceph-mon[64115]: pgmap v309: 97 pgs: 97 active+clean; 329 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 9.1 MiB/s rd, 5.1 MiB/s wr, 107 op/s; 439 KiB/s, 4 objects/s recovering 2024-09-16T08:25:14.106 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:25:13 smithi028 ceph-mon[64115]: from='client.34472 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T08:25:14.106 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:25:13 smithi028 ceph-mon[64115]: from='client.? 172.21.15.28:0/3980625619' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-16T08:25:14.242 INFO:teuthology.orchestra.run.smithi028.stdout:{ 2024-09-16T08:25:14.242 INFO:teuthology.orchestra.run.smithi028.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:f7ad55f9ccd17ba66aadb19b4d291e67b60ea24e5d8a0c0e92941fd59c81e4ce", 2024-09-16T08:25:14.242 INFO:teuthology.orchestra.run.smithi028.stdout: "in_progress": true, 2024-09-16T08:25:14.242 INFO:teuthology.orchestra.run.smithi028.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-16T08:25:14.243 INFO:teuthology.orchestra.run.smithi028.stdout: "services_complete": [ 2024-09-16T08:25:14.243 INFO:teuthology.orchestra.run.smithi028.stdout: "crash", 2024-09-16T08:25:14.243 INFO:teuthology.orchestra.run.smithi028.stdout: "mon", 2024-09-16T08:25:14.243 INFO:teuthology.orchestra.run.smithi028.stdout: "osd", 2024-09-16T08:25:14.243 INFO:teuthology.orchestra.run.smithi028.stdout: "mgr" 2024-09-16T08:25:14.243 INFO:teuthology.orchestra.run.smithi028.stdout: ], 2024-09-16T08:25:14.243 INFO:teuthology.orchestra.run.smithi028.stdout: "progress": "14/23 daemons upgraded", 2024-09-16T08:25:14.243 INFO:teuthology.orchestra.run.smithi028.stdout: "message": "Currently upgrading mds daemons", 2024-09-16T08:25:14.243 INFO:teuthology.orchestra.run.smithi028.stdout: "is_paused": false 2024-09-16T08:25:14.243 INFO:teuthology.orchestra.run.smithi028.stdout:} 2024-09-16T08:25:14.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:25:13 smithi144 ceph-mon[49149]: from='client.44409 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T08:25:14.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:25:13 smithi144 ceph-mon[49149]: pgmap v309: 97 pgs: 97 active+clean; 329 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 9.1 MiB/s rd, 5.1 MiB/s wr, 107 op/s; 439 KiB/s, 4 objects/s recovering 2024-09-16T08:25:14.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:25:13 smithi144 ceph-mon[49149]: from='client.34472 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T08:25:14.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:25:13 smithi144 ceph-mon[49149]: from='client.? 172.21.15.28:0/3980625619' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-16T08:25:14.431 INFO:teuthology.orchestra.run.smithi028.stdout: 5 1011 17.29 MB/sec execute 4 sec latency 42.329 ms 2024-09-16T08:25:14.948 INFO:teuthology.orchestra.run.smithi028.stdout:HEALTH_OK 2024-09-16T08:25:15.125 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:25:14 smithi028 ceph-mon[64115]: from='client.? 172.21.15.28:0/1469914194' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-16T08:25:15.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:25:14 smithi144 ceph-mon[49149]: from='client.? 172.21.15.28:0/1469914194' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-16T08:25:15.431 INFO:teuthology.orchestra.run.smithi028.stdout: 5 cleanup 5 sec 2024-09-16T08:25:15.545 INFO:teuthology.orchestra.run.smithi028.stdout:[1387] unlink './clients/client1/~dmtmp/PARADOX/.nfs000001000000bde30000001d' failed - Device or resource busy 2024-09-16T08:25:15.545 INFO:teuthology.orchestra.run.smithi028.stdout:[1387] unlink './clients/client1/~dmtmp/PARADOX/.nfs000001000000bde800000019' failed - Device or resource busy 2024-09-16T08:25:15.629 INFO:teuthology.orchestra.run.smithi028.stdout: 0 cleanup 5 sec 2024-09-16T08:25:15.629 INFO:teuthology.orchestra.run.smithi028.stdout: 2024-09-16T08:25:15.630 INFO:teuthology.orchestra.run.smithi028.stdout: Operation Count AvgLat MaxLat 2024-09-16T08:25:15.630 INFO:teuthology.orchestra.run.smithi028.stdout: ---------------------------------------- 2024-09-16T08:25:15.630 INFO:teuthology.orchestra.run.smithi028.stdout: NTCreateX 804 8.342 38.147 2024-09-16T08:25:15.630 INFO:teuthology.orchestra.run.smithi028.stdout: Close 755 7.255 31.175 2024-09-16T08:25:15.630 INFO:teuthology.orchestra.run.smithi028.stdout: Rename 50 14.140 42.318 2024-09-16T08:25:15.630 INFO:teuthology.orchestra.run.smithi028.stdout: Unlink 88 4.099 18.416 2024-09-16T08:25:15.630 INFO:teuthology.orchestra.run.smithi028.stdout: Qpathinfo 488 1.806 20.546 2024-09-16T08:25:15.630 INFO:teuthology.orchestra.run.smithi028.stdout: Qfileinfo 349 17.690 36.354 2024-09-16T08:25:15.630 INFO:teuthology.orchestra.run.smithi028.stdout: Qfsinfo 165 0.714 4.407 2024-09-16T08:25:15.630 INFO:teuthology.orchestra.run.smithi028.stdout: Sfileinfo 170 14.390 39.841 2024-09-16T08:25:15.630 INFO:teuthology.orchestra.run.smithi028.stdout: Find 231 1.197 10.696 2024-09-16T08:25:15.631 INFO:teuthology.orchestra.run.smithi028.stdout: WriteX 1639 0.027 0.082 2024-09-16T08:25:15.631 INFO:teuthology.orchestra.run.smithi028.stdout: ReadX 1025 0.021 3.311 2024-09-16T08:25:15.631 INFO:teuthology.orchestra.run.smithi028.stdout: Flush 120 15.214 24.388 2024-09-16T08:25:15.631 INFO:teuthology.orchestra.run.smithi028.stdout: 2024-09-16T08:25:15.631 INFO:teuthology.orchestra.run.smithi028.stdout:Throughput 17.2872 MB/sec 5 clients 5 procs max_latency=56.901 ms 2024-09-16T08:25:15.632 DEBUG:teuthology.parallel:result is None 2024-09-16T08:25:16.286 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:25:15 smithi028 ceph-mon[64115]: from='client.44421 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T08:25:16.286 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:25:15 smithi028 ceph-mon[64115]: pgmap v310: 97 pgs: 97 active+clean; 330 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4.0 MiB/s rd, 8.3 MiB/s wr, 288 op/s 2024-09-16T08:25:16.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:25:15 smithi144 ceph-mon[49149]: from='client.44421 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T08:25:16.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:25:15 smithi144 ceph-mon[49149]: pgmap v310: 97 pgs: 97 active+clean; 330 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4.0 MiB/s rd, 8.3 MiB/s wr, 288 op/s 2024-09-16T08:25:18.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:25:18 smithi144 ceph-mon[49149]: pgmap v311: 97 pgs: 97 active+clean; 332 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 2.0 MiB/s rd, 11 MiB/s wr, 314 op/s 2024-09-16T08:25:18.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:25:18 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:25:18.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:25:18 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-16T08:25:18.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:25:18 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:25:18.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:25:18 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:25:18.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:25:18 smithi028 ceph-mon[64115]: pgmap v311: 97 pgs: 97 active+clean; 332 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 2.0 MiB/s rd, 11 MiB/s wr, 314 op/s 2024-09-16T08:25:18.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:25:18 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:25:18.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:25:18 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-16T08:25:18.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:25:18 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:25:18.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:25:18 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:25:19.697 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:25:19 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:25:19.697 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:25:19 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:25:19.697 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:25:19 smithi028 ceph-mon[64115]: pgmap v312: 97 pgs: 97 active+clean; 328 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 35 KiB/s rd, 12 MiB/s wr, 329 op/s 2024-09-16T08:25:19.832 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:25:19 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:25:19.832 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:25:19 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:25:19.832 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:25:19 smithi144 ceph-mon[49149]: pgmap v312: 97 pgs: 97 active+clean; 328 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 35 KiB/s rd, 12 MiB/s wr, 329 op/s 2024-09-16T08:25:22.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:25:21 smithi144 ceph-mon[49149]: pgmap v313: 97 pgs: 97 active+clean; 290 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 21 KiB/s rd, 11 MiB/s wr, 328 op/s 2024-09-16T08:25:22.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:25:21 smithi028 ceph-mon[64115]: pgmap v313: 97 pgs: 97 active+clean; 290 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 21 KiB/s rd, 11 MiB/s wr, 328 op/s 2024-09-16T08:25:23.227 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:25:23 smithi144 ceph-mon[49149]: Detected new or changed devices on smithi028 2024-09-16T08:25:23.227 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:25:23 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:25:23.227 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:25:23 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:25:23.227 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:25:23 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-16T08:25:23.227 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:25:23 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-16T08:25:23.228 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:25:23 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:25:23.228 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:25:23 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:25:23.228 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:25:23 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-16T08:25:23.228 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:25:23 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "versions"} : dispatch 2024-09-16T08:25:23.228 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:25:23 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "versions"} : dispatch 2024-09-16T08:25:23.228 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:25:23 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "versions"} : dispatch 2024-09-16T08:25:23.228 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:25:23 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "versions"} : dispatch 2024-09-16T08:25:23.228 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:25:23 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "mds ok-to-stop", "ids": ["foofs.smithi144.oozgsi"]} : dispatch 2024-09-16T08:25:23.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:25:23 smithi028 ceph-mon[64115]: Detected new or changed devices on smithi028 2024-09-16T08:25:23.448 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:25:23 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:25:23.448 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:25:23 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:25:23.448 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:25:23 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-16T08:25:23.448 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:25:23 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-16T08:25:23.448 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:25:23 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:25:23.448 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:25:23 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:25:23.448 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:25:23 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-16T08:25:23.448 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:25:23 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "versions"} : dispatch 2024-09-16T08:25:23.448 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:25:23 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "versions"} : dispatch 2024-09-16T08:25:23.448 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:25:23 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "versions"} : dispatch 2024-09-16T08:25:23.448 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:25:23 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "versions"} : dispatch 2024-09-16T08:25:23.449 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:25:23 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "mds ok-to-stop", "ids": ["foofs.smithi144.oozgsi"]} : dispatch 2024-09-16T08:25:24.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:25:24 smithi028 ceph-mon[64115]: Upgrade: It appears safe to stop mds.foofs.smithi144.oozgsi 2024-09-16T08:25:24.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:25:24 smithi028 ceph-mon[64115]: pgmap v314: 97 pgs: 97 active+clean; 243 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 15 KiB/s rd, 9.0 MiB/s wr, 294 op/s 2024-09-16T08:25:24.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:25:24 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:25:24.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:25:24 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "auth get-or-create", "entity": "mds.foofs.smithi144.oozgsi", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-09-16T08:25:24.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:25:24 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-16T08:25:24.479 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:25:24 smithi144 ceph-mon[49149]: Upgrade: It appears safe to stop mds.foofs.smithi144.oozgsi 2024-09-16T08:25:24.479 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:25:24 smithi144 ceph-mon[49149]: pgmap v314: 97 pgs: 97 active+clean; 243 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 15 KiB/s rd, 9.0 MiB/s wr, 294 op/s 2024-09-16T08:25:24.479 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:25:24 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:25:24.479 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:25:24 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "auth get-or-create", "entity": "mds.foofs.smithi144.oozgsi", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-09-16T08:25:24.479 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:25:24 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-16T08:25:25.182 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:25:25 smithi144 ceph-mon[49149]: Upgrade: Updating mds.foofs.smithi144.oozgsi 2024-09-16T08:25:25.183 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:25:25 smithi144 ceph-mon[49149]: Deploying daemon mds.foofs.smithi144.oozgsi on smithi144 2024-09-16T08:25:25.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:25:25 smithi028 ceph-mon[64115]: Upgrade: Updating mds.foofs.smithi144.oozgsi 2024-09-16T08:25:25.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:25:25 smithi028 ceph-mon[64115]: Deploying daemon mds.foofs.smithi144.oozgsi on smithi144 2024-09-16T08:25:25.954 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:25:25 smithi028 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-mon-smithi028[64111]: 2024-09-16T08:25:25.680+0000 7f332caa2640 -1 log_channel(cluster) log [ERR] : Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-09-16T08:25:26.082 INFO:journalctl@ceph.osd.5.smithi144.stdout:Sep 16 08:25:25 smithi144 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-5[55067]: 2024-09-16T08:25:25.710+0000 7fdc50d90640 -1 bdev(0x559e0c666c00 /var/lib/ceph/osd/ceph-5/block) aio_submit retries 2 2024-09-16T08:25:26.083 INFO:journalctl@ceph.osd.5.smithi144.stdout:Sep 16 08:25:25 smithi144 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-5[55067]: 2024-09-16T08:25:25.717+0000 7fdc51d92640 -1 bdev(0x559e0c666c00 /var/lib/ceph/osd/ceph-5/block) aio_submit retries 5 2024-09-16T08:25:26.083 INFO:journalctl@ceph.osd.5.smithi144.stdout:Sep 16 08:25:25 smithi144 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-5[55067]: 2024-09-16T08:25:25.718+0000 7fdc4f58d640 -1 bdev(0x559e0c666c00 /var/lib/ceph/osd/ceph-5/block) aio_submit retries 6 2024-09-16T08:25:26.083 INFO:journalctl@ceph.osd.6.smithi144.stdout:Sep 16 08:25:25 smithi144 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-6[57105]: 2024-09-16T08:25:25.707+0000 7fc03801d640 -1 bdev(0x555b93530c00 /var/lib/ceph/osd/ceph-6/block) aio_submit retries 1 2024-09-16T08:25:26.083 INFO:journalctl@ceph.osd.6.smithi144.stdout:Sep 16 08:25:25 smithi144 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-6[57105]: 2024-09-16T08:25:25.707+0000 7fc03c025640 -1 bdev(0x555b93530c00 /var/lib/ceph/osd/ceph-6/block) aio_submit retries 2 2024-09-16T08:25:26.083 INFO:journalctl@ceph.osd.7.smithi144.stdout:Sep 16 08:25:25 smithi144 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-7[59146]: 2024-09-16T08:25:25.710+0000 7f4a83a05640 -1 bdev(0x5561662e4c00 /var/lib/ceph/osd/ceph-7/block) aio_submit retries 1 2024-09-16T08:25:26.162 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:25:26 smithi028 ceph-mon[64115]: pgmap v315: 97 pgs: 97 active+clean; 235 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 4.4 KiB/s rd, 5.7 MiB/s wr, 246 op/s 2024-09-16T08:25:26.162 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:25:26 smithi028 ceph-mon[64115]: Health check failed: 1 filesystem is degraded (FS_DEGRADED) 2024-09-16T08:25:26.162 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:25:26 smithi028 ceph-mon[64115]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-09-16T08:25:26.162 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:25:26 smithi028 ceph-mon[64115]: osdmap e156: 8 total, 8 up, 8 in 2024-09-16T08:25:26.162 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:25:26 smithi028 ceph-mon[64115]: Standby daemon mds.foofs.smithi028.srtgev assigned to filesystem foofs as rank 0 2024-09-16T08:25:26.162 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:25:26 smithi028 ceph-mon[64115]: Health check failed: insufficient standby MDS daemons available (MDS_INSUFFICIENT_STANDBY) 2024-09-16T08:25:26.162 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:25:26 smithi028 ceph-mon[64115]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-09-16T08:25:26.162 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:25:26 smithi028 ceph-mon[64115]: fsmap foofs:0/1 1 up:standby, 1 failed 2024-09-16T08:25:26.162 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:25:26 smithi028 ceph-mon[64115]: fsmap foofs:1/1 {0=foofs.smithi028.srtgev=up:replay} 2024-09-16T08:25:26.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:25:26 smithi144 ceph-mon[49149]: pgmap v315: 97 pgs: 97 active+clean; 235 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 4.4 KiB/s rd, 5.7 MiB/s wr, 246 op/s 2024-09-16T08:25:26.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:25:26 smithi144 ceph-mon[49149]: Health check failed: 1 filesystem is degraded (FS_DEGRADED) 2024-09-16T08:25:26.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:25:26 smithi144 ceph-mon[49149]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-09-16T08:25:26.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:25:26 smithi144 ceph-mon[49149]: osdmap e156: 8 total, 8 up, 8 in 2024-09-16T08:25:26.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:25:26 smithi144 ceph-mon[49149]: Standby daemon mds.foofs.smithi028.srtgev assigned to filesystem foofs as rank 0 2024-09-16T08:25:26.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:25:26 smithi144 ceph-mon[49149]: Health check failed: insufficient standby MDS daemons available (MDS_INSUFFICIENT_STANDBY) 2024-09-16T08:25:26.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:25:26 smithi144 ceph-mon[49149]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-09-16T08:25:26.583 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:25:26 smithi144 ceph-mon[49149]: fsmap foofs:0/1 1 up:standby, 1 failed 2024-09-16T08:25:26.583 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:25:26 smithi144 ceph-mon[49149]: fsmap foofs:1/1 {0=foofs.smithi028.srtgev=up:replay} 2024-09-16T08:25:28.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:25:28 smithi028 ceph-mon[64115]: pgmap v317: 97 pgs: 97 active+clean; 232 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 7.2 MiB/s rd, 1.3 MiB/s wr, 85 op/s 2024-09-16T08:25:28.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:25:28 smithi028 ceph-mon[64115]: reconnect by client.14546 172.21.15.28:0/1295173080 after 0.000999973 2024-09-16T08:25:28.448 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:25:28 smithi028 ceph-mon[64115]: reconnect by client.24359 172.21.15.144:0/2699919051 after 0.000999973 2024-09-16T08:25:28.448 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:25:28 smithi028 ceph-mon[64115]: mds.? [v2:172.21.15.28:6834/1804746814,v1:172.21.15.28:6835/1804746814] up:reconnect 2024-09-16T08:25:28.448 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:25:28 smithi028 ceph-mon[64115]: fsmap foofs:1/1 {0=foofs.smithi028.srtgev=up:reconnect} 2024-09-16T08:25:28.448 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:25:28 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "mds metadata", "who": "foofs.smithi144.oozgsi"} : dispatch 2024-09-16T08:25:28.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:25:28 smithi144 ceph-mon[49149]: pgmap v317: 97 pgs: 97 active+clean; 232 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 7.2 MiB/s rd, 1.3 MiB/s wr, 85 op/s 2024-09-16T08:25:28.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:25:28 smithi144 ceph-mon[49149]: reconnect by client.14546 172.21.15.28:0/1295173080 after 0.000999973 2024-09-16T08:25:28.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:25:28 smithi144 ceph-mon[49149]: reconnect by client.24359 172.21.15.144:0/2699919051 after 0.000999973 2024-09-16T08:25:28.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:25:28 smithi144 ceph-mon[49149]: mds.? [v2:172.21.15.28:6834/1804746814,v1:172.21.15.28:6835/1804746814] up:reconnect 2024-09-16T08:25:28.583 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:25:28 smithi144 ceph-mon[49149]: fsmap foofs:1/1 {0=foofs.smithi028.srtgev=up:reconnect} 2024-09-16T08:25:28.583 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:25:28 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "mds metadata", "who": "foofs.smithi144.oozgsi"} : dispatch 2024-09-16T08:25:29.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:25:29 smithi028 ceph-mon[64115]: mds.? [v2:172.21.15.28:6834/1804746814,v1:172.21.15.28:6835/1804746814] up:rejoin 2024-09-16T08:25:29.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:25:29 smithi028 ceph-mon[64115]: fsmap foofs:1/1 {0=foofs.smithi028.srtgev=up:rejoin} 2024-09-16T08:25:29.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:25:29 smithi028 ceph-mon[64115]: daemon mds.foofs.smithi028.srtgev is now active in filesystem foofs as rank 0 2024-09-16T08:25:29.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:25:29 smithi144 ceph-mon[49149]: mds.? [v2:172.21.15.28:6834/1804746814,v1:172.21.15.28:6835/1804746814] up:rejoin 2024-09-16T08:25:29.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:25:29 smithi144 ceph-mon[49149]: fsmap foofs:1/1 {0=foofs.smithi028.srtgev=up:rejoin} 2024-09-16T08:25:29.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:25:29 smithi144 ceph-mon[49149]: daemon mds.foofs.smithi028.srtgev is now active in filesystem foofs as rank 0 2024-09-16T08:25:30.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:25:30 smithi028 ceph-mon[64115]: pgmap v318: 97 pgs: 97 active+clean; 232 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 16 MiB/s rd, 286 KiB/s wr, 62 op/s 2024-09-16T08:25:30.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:25:30 smithi028 ceph-mon[64115]: Health check cleared: FS_DEGRADED (was: 1 filesystem is degraded) 2024-09-16T08:25:30.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:25:30 smithi028 ceph-mon[64115]: mds.? [v2:172.21.15.28:6834/1804746814,v1:172.21.15.28:6835/1804746814] up:active 2024-09-16T08:25:30.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:25:30 smithi028 ceph-mon[64115]: fsmap foofs:1 {0=foofs.smithi028.srtgev=up:active} 2024-09-16T08:25:30.483 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:25:30 smithi144 ceph-mon[49149]: pgmap v318: 97 pgs: 97 active+clean; 232 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 16 MiB/s rd, 286 KiB/s wr, 62 op/s 2024-09-16T08:25:30.483 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:25:30 smithi144 ceph-mon[49149]: Health check cleared: FS_DEGRADED (was: 1 filesystem is degraded) 2024-09-16T08:25:30.483 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:25:30 smithi144 ceph-mon[49149]: mds.? [v2:172.21.15.28:6834/1804746814,v1:172.21.15.28:6835/1804746814] up:active 2024-09-16T08:25:30.484 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:25:30 smithi144 ceph-mon[49149]: fsmap foofs:1 {0=foofs.smithi028.srtgev=up:active} 2024-09-16T08:25:32.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:25:32 smithi028 ceph-mon[64115]: pgmap v319: 97 pgs: 97 active+clean; 232 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 23 MiB/s rd, 5.8 KiB/s wr, 25 op/s 2024-09-16T08:25:32.548 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:25:32 smithi144 ceph-mon[49149]: pgmap v319: 97 pgs: 97 active+clean; 232 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 23 MiB/s rd, 5.8 KiB/s wr, 25 op/s 2024-09-16T08:25:33.570 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:25:33 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:25:33.570 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:25:33 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-16T08:25:33.570 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:25:33 smithi144 ceph-mon[49149]: pgmap v320: 97 pgs: 97 active+clean; 232 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 23 MiB/s rd, 2.2 KiB/s wr, 11 op/s 2024-09-16T08:25:33.697 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:25:33 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:25:33.697 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:25:33 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-16T08:25:33.697 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:25:33 smithi028 ceph-mon[64115]: pgmap v320: 97 pgs: 97 active+clean; 232 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 23 MiB/s rd, 2.2 KiB/s wr, 11 op/s 2024-09-16T08:25:34.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:25:34 smithi144 ceph-mon[49149]: Health check cleared: MDS_INSUFFICIENT_STANDBY (was: insufficient standby MDS daemons available) 2024-09-16T08:25:34.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:25:34 smithi144 ceph-mon[49149]: Cluster is now healthy 2024-09-16T08:25:34.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:25:34 smithi144 ceph-mon[49149]: mds.? [v2:172.21.15.144:6832/2338792151,v1:172.21.15.144:6833/2338792151] up:boot 2024-09-16T08:25:34.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:25:34 smithi144 ceph-mon[49149]: fsmap foofs:1 {0=foofs.smithi028.srtgev=up:active} 1 up:standby 2024-09-16T08:25:34.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:25:34 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "mds metadata", "who": "foofs.smithi144.oozgsi"} : dispatch 2024-09-16T08:25:34.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:25:34 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:25:34.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:25:34 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:25:34.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:25:34 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-16T08:25:34.697 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:25:34 smithi028 ceph-mon[64115]: Health check cleared: MDS_INSUFFICIENT_STANDBY (was: insufficient standby MDS daemons available) 2024-09-16T08:25:34.697 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:25:34 smithi028 ceph-mon[64115]: Cluster is now healthy 2024-09-16T08:25:34.697 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:25:34 smithi028 ceph-mon[64115]: mds.? [v2:172.21.15.144:6832/2338792151,v1:172.21.15.144:6833/2338792151] up:boot 2024-09-16T08:25:34.697 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:25:34 smithi028 ceph-mon[64115]: fsmap foofs:1 {0=foofs.smithi028.srtgev=up:active} 1 up:standby 2024-09-16T08:25:34.698 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:25:34 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "mds metadata", "who": "foofs.smithi144.oozgsi"} : dispatch 2024-09-16T08:25:34.698 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:25:34 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:25:34.698 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:25:34 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:25:34.698 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:25:34 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-16T08:25:35.614 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:25:35 smithi144 ceph-mon[49149]: pgmap v321: 97 pgs: 97 active+clean; 232 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 23 MiB/s rd, 8.5 KiB/s wr, 15 op/s 2024-09-16T08:25:35.697 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:25:35 smithi028 ceph-mon[64115]: pgmap v321: 97 pgs: 97 active+clean; 232 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 23 MiB/s rd, 8.5 KiB/s wr, 15 op/s 2024-09-16T08:25:38.264 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:25:37 smithi144 ceph-mon[49149]: pgmap v322: 97 pgs: 97 active+clean; 232 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 20 MiB/s rd, 7.6 KiB/s wr, 13 op/s 2024-09-16T08:25:38.264 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:25:37 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:25:38.265 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:25:37 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:25:38.265 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:25:37 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:25:38.265 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:25:37 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:25:38.449 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:25:37 smithi028 ceph-mon[64115]: pgmap v322: 97 pgs: 97 active+clean; 232 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 20 MiB/s rd, 7.6 KiB/s wr, 13 op/s 2024-09-16T08:25:38.450 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:25:37 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:25:38.450 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:25:37 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:25:38.450 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:25:37 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:25:38.450 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:25:37 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:25:40.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:25:39 smithi144 ceph-mon[49149]: pgmap v323: 97 pgs: 97 active+clean; 232 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 13 MiB/s rd, 7.0 KiB/s wr, 10 op/s 2024-09-16T08:25:40.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:25:39 smithi028 ceph-mon[64115]: pgmap v323: 97 pgs: 97 active+clean; 232 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 13 MiB/s rd, 7.0 KiB/s wr, 10 op/s 2024-09-16T08:25:41.871 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:25:41 smithi028 ceph-mon[64115]: Detected new or changed devices on smithi144 2024-09-16T08:25:41.871 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:25:41 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:25:41.871 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:25:41 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:25:41.871 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:25:41 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-16T08:25:41.871 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:25:41 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-16T08:25:41.871 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:25:41 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:25:41.872 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:25:41 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:25:41.872 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:25:41 smithi028 ceph-mon[64115]: pgmap v324: 97 pgs: 97 active+clean; 232 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 5.6 MiB/s rd, 7.5 KiB/s wr, 8 op/s 2024-09-16T08:25:41.872 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:25:41 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-16T08:25:41.872 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:25:41 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "versions"} : dispatch 2024-09-16T08:25:41.872 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:25:41 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "versions"} : dispatch 2024-09-16T08:25:41.872 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:25:41 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "versions"} : dispatch 2024-09-16T08:25:41.872 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:25:41 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "versions"} : dispatch 2024-09-16T08:25:41.872 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:25:41 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "versions"} : dispatch 2024-09-16T08:25:41.872 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:25:41 smithi028 ceph-mon[64115]: Upgrade: Setting container_image for all mds 2024-09-16T08:25:41.872 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:25:41 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:25:41.872 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:25:41 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi028.srtgev"} : dispatch 2024-09-16T08:25:41.872 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:25:41 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi028.srtgev"}]': finished 2024-09-16T08:25:41.872 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:25:41 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi144.oozgsi"} : dispatch 2024-09-16T08:25:41.873 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:25:41 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi144.oozgsi"}]': finished 2024-09-16T08:25:41.873 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:25:41 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "versions"} : dispatch 2024-09-16T08:25:41.873 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:25:41 smithi028 ceph-mon[64115]: Upgrade: Setting container_image for all rgw 2024-09-16T08:25:41.873 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:25:41 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:25:41.873 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:25:41 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "versions"} : dispatch 2024-09-16T08:25:41.873 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:25:41 smithi028 ceph-mon[64115]: Upgrade: Setting container_image for all rbd-mirror 2024-09-16T08:25:41.873 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:25:41 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:25:41.873 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:25:41 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "versions"} : dispatch 2024-09-16T08:25:41.873 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:25:41 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "versions"} : dispatch 2024-09-16T08:25:41.873 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:25:41 smithi028 ceph-mon[64115]: Upgrade: Setting container_image for all ceph-exporter 2024-09-16T08:25:41.874 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:25:41 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:25:41.874 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:25:41 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "versions"} : dispatch 2024-09-16T08:25:41.874 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:25:41 smithi028 ceph-mon[64115]: Upgrade: Setting container_image for all iscsi 2024-09-16T08:25:41.874 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:25:41 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:25:42.082 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:25:41 smithi144 ceph-mon[49149]: Detected new or changed devices on smithi144 2024-09-16T08:25:42.082 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:25:41 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:25:42.083 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:25:41 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:25:42.083 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:25:41 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-16T08:25:42.083 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:25:41 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-16T08:25:42.083 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:25:41 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:25:42.083 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:25:41 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:25:42.083 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:25:41 smithi144 ceph-mon[49149]: pgmap v324: 97 pgs: 97 active+clean; 232 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 5.6 MiB/s rd, 7.5 KiB/s wr, 8 op/s 2024-09-16T08:25:42.083 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:25:41 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-16T08:25:42.083 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:25:41 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "versions"} : dispatch 2024-09-16T08:25:42.083 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:25:41 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "versions"} : dispatch 2024-09-16T08:25:42.083 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:25:41 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "versions"} : dispatch 2024-09-16T08:25:42.083 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:25:41 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "versions"} : dispatch 2024-09-16T08:25:42.083 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:25:41 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "versions"} : dispatch 2024-09-16T08:25:42.084 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:25:41 smithi144 ceph-mon[49149]: Upgrade: Setting container_image for all mds 2024-09-16T08:25:42.084 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:25:41 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:25:42.084 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:25:41 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi028.srtgev"} : dispatch 2024-09-16T08:25:42.084 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:25:41 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi028.srtgev"}]': finished 2024-09-16T08:25:42.084 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:25:41 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi144.oozgsi"} : dispatch 2024-09-16T08:25:42.084 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:25:41 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi144.oozgsi"}]': finished 2024-09-16T08:25:42.084 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:25:41 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "versions"} : dispatch 2024-09-16T08:25:42.084 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:25:41 smithi144 ceph-mon[49149]: Upgrade: Setting container_image for all rgw 2024-09-16T08:25:42.084 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:25:41 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:25:42.084 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:25:41 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "versions"} : dispatch 2024-09-16T08:25:42.084 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:25:41 smithi144 ceph-mon[49149]: Upgrade: Setting container_image for all rbd-mirror 2024-09-16T08:25:42.084 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:25:41 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:25:42.084 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:25:41 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "versions"} : dispatch 2024-09-16T08:25:42.085 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:25:41 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "versions"} : dispatch 2024-09-16T08:25:42.085 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:25:41 smithi144 ceph-mon[49149]: Upgrade: Setting container_image for all ceph-exporter 2024-09-16T08:25:42.085 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:25:41 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:25:42.085 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:25:41 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "versions"} : dispatch 2024-09-16T08:25:42.085 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:25:41 smithi144 ceph-mon[49149]: Upgrade: Setting container_image for all iscsi 2024-09-16T08:25:42.085 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:25:41 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:25:43.947 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:25:43 smithi028 ceph-mon[64115]: Upgrade: Updating nfs.foo.0.0.smithi028.dkttgj (1/2) 2024-09-16T08:25:43.948 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:25:43 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:25:43.948 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:25:43 smithi028 ceph-mon[64115]: Creating key for client.nfs.foo.0.0.smithi028.dkttgj 2024-09-16T08:25:43.948 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:25:43 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi028.dkttgj", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]} : dispatch 2024-09-16T08:25:43.948 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:25:43 smithi028 ceph-mon[64115]: Ensuring nfs.foo.0 is in the ganesha grace table 2024-09-16T08:25:43.948 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:25:43 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]} : dispatch 2024-09-16T08:25:43.948 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:25:43 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-09-16T08:25:43.948 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:25:43 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-16T08:25:43.948 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:25:43 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"} : dispatch 2024-09-16T08:25:43.948 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:25:43 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-09-16T08:25:43.948 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:25:43 smithi028 ceph-mon[64115]: Rados config object exists: conf-nfs.foo 2024-09-16T08:25:43.948 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:25:43 smithi028 ceph-mon[64115]: Creating key for client.nfs.foo.0.0.smithi028.dkttgj-rgw 2024-09-16T08:25:43.948 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:25:43 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi028.dkttgj-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]} : dispatch 2024-09-16T08:25:43.949 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:25:43 smithi028 ceph-mon[64115]: Bind address in nfs.foo.0.0.smithi028.dkttgj's ganesha conf is defaulting to empty 2024-09-16T08:25:43.949 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:25:43 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-16T08:25:43.949 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:25:43 smithi028 ceph-mon[64115]: Deploying daemon nfs.foo.0.0.smithi028.dkttgj on smithi028 2024-09-16T08:25:43.949 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:25:43 smithi028 ceph-mon[64115]: pgmap v325: 97 pgs: 97 active+clean; 232 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 3.3 KiB/s rd, 7.4 KiB/s wr, 5 op/s 2024-09-16T08:25:44.082 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:25:43 smithi144 ceph-mon[49149]: Upgrade: Updating nfs.foo.0.0.smithi028.dkttgj (1/2) 2024-09-16T08:25:44.082 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:25:43 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:25:44.082 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:25:43 smithi144 ceph-mon[49149]: Creating key for client.nfs.foo.0.0.smithi028.dkttgj 2024-09-16T08:25:44.083 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:25:43 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi028.dkttgj", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]} : dispatch 2024-09-16T08:25:44.083 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:25:43 smithi144 ceph-mon[49149]: Ensuring nfs.foo.0 is in the ganesha grace table 2024-09-16T08:25:44.083 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:25:43 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]} : dispatch 2024-09-16T08:25:44.083 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:25:43 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-09-16T08:25:44.083 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:25:43 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-16T08:25:44.083 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:25:43 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"} : dispatch 2024-09-16T08:25:44.083 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:25:43 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-09-16T08:25:44.083 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:25:43 smithi144 ceph-mon[49149]: Rados config object exists: conf-nfs.foo 2024-09-16T08:25:44.083 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:25:43 smithi144 ceph-mon[49149]: Creating key for client.nfs.foo.0.0.smithi028.dkttgj-rgw 2024-09-16T08:25:44.083 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:25:43 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi028.dkttgj-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]} : dispatch 2024-09-16T08:25:44.083 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:25:43 smithi144 ceph-mon[49149]: Bind address in nfs.foo.0.0.smithi028.dkttgj's ganesha conf is defaulting to empty 2024-09-16T08:25:44.083 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:25:43 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-16T08:25:44.083 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:25:43 smithi144 ceph-mon[49149]: Deploying daemon nfs.foo.0.0.smithi028.dkttgj on smithi028 2024-09-16T08:25:44.083 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:25:43 smithi144 ceph-mon[49149]: pgmap v325: 97 pgs: 97 active+clean; 232 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 3.3 KiB/s rd, 7.4 KiB/s wr, 5 op/s 2024-09-16T08:25:45.379 INFO:teuthology.orchestra.run.smithi028.stdout:true 2024-09-16T08:25:46.030 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:25:45 smithi028 ceph-mon[64115]: pgmap v326: 97 pgs: 97 active+clean; 232 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 2.3 KiB/s rd, 7.7 KiB/s wr, 5 op/s 2024-09-16T08:25:46.164 INFO:teuthology.orchestra.run.smithi028.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-16T08:25:46.165 INFO:teuthology.orchestra.run.smithi028.stdout:alertmanager.smithi028 smithi028 *:9093,9094 running (10m) 28s ago 19m 22.2M - 0.27.0 11f11916f8cd a14fcb4cb2b6 2024-09-16T08:25:46.165 INFO:teuthology.orchestra.run.smithi028.stdout:crash.smithi028 smithi028 running (6m) 28s ago 19m 6899k - 19.3.0-4917-g8293d73f 425e316ba8eb 4bb89df2a867 2024-09-16T08:25:46.165 INFO:teuthology.orchestra.run.smithi028.stdout:crash.smithi144 smithi144 running (6m) 9s ago 18m 6895k - 19.3.0-4917-g8293d73f 425e316ba8eb 24a96875d7af 2024-09-16T08:25:46.165 INFO:teuthology.orchestra.run.smithi028.stdout:grafana.smithi028 smithi028 *:3000 running (9m) 28s ago 18m 54.1M - dad864ee21e9 b55ba8b944a1 2024-09-16T08:25:46.165 INFO:teuthology.orchestra.run.smithi028.stdout:mds.foofs.smithi028.srtgev smithi028 running (35s) 28s ago 13m 13.1M - 19.3.0-4917-g8293d73f 425e316ba8eb 102d36a11737 2024-09-16T08:25:46.165 INFO:teuthology.orchestra.run.smithi028.stdout:mds.foofs.smithi144.oozgsi smithi144 running (12s) 9s ago 13m 12.9M - 19.3.0-4917-g8293d73f 425e316ba8eb 9917dfd27777 2024-09-16T08:25:46.165 INFO:teuthology.orchestra.run.smithi028.stdout:mgr.smithi028.azvyzy smithi028 *:8443,9283,8765 running (9m) 28s ago 20m 523M - 19.3.0-4917-g8293d73f 425e316ba8eb e01d4382d03f 2024-09-16T08:25:46.165 INFO:teuthology.orchestra.run.smithi028.stdout:mgr.smithi144.utzszj smithi144 *:8443,9283,8765 running (8m) 9s ago 18m 458M - 19.3.0-4917-g8293d73f 425e316ba8eb 71f7f802bffb 2024-09-16T08:25:46.165 INFO:teuthology.orchestra.run.smithi028.stdout:mon.smithi028 smithi028 running (8m) 28s ago 20m 60.7M 2048M 19.3.0-4917-g8293d73f 425e316ba8eb 14efcedc0088 2024-09-16T08:25:46.165 INFO:teuthology.orchestra.run.smithi028.stdout:mon.smithi144 smithi144 running (7m) 9s ago 18m 50.5M 2048M 19.3.0-4917-g8293d73f 425e316ba8eb c081c5ed44b5 2024-09-16T08:25:46.166 INFO:teuthology.orchestra.run.smithi028.stdout:nfs.foo.0.0.smithi028.dkttgj smithi028 *:2049 running (12m) 28s ago 12m 302M - 4.0 e1d6a67b021e b5dccc45a6ed 2024-09-16T08:25:46.166 INFO:teuthology.orchestra.run.smithi028.stdout:nfs.foo.1.0.smithi144.epipkg smithi144 *:2049 running (12m) 9s ago 12m 87.5M - 4.0 e1d6a67b021e faa89e61f50e 2024-09-16T08:25:46.166 INFO:teuthology.orchestra.run.smithi028.stdout:node-exporter.smithi028 smithi028 *:9100 running (9m) 28s ago 18m 11.5M - 1.7.0 72c9c2088986 9e2b8e8a63a9 2024-09-16T08:25:46.166 INFO:teuthology.orchestra.run.smithi028.stdout:node-exporter.smithi144 smithi144 *:9100 running (9m) 9s ago 18m 10.5M - 1.7.0 72c9c2088986 78834c5e992f 2024-09-16T08:25:46.166 INFO:teuthology.orchestra.run.smithi028.stdout:osd.0 smithi028 running (6m) 28s ago 17m 419M 1937M 19.3.0-4917-g8293d73f 425e316ba8eb 2fc02ba4f14f 2024-09-16T08:25:46.166 INFO:teuthology.orchestra.run.smithi028.stdout:osd.1 smithi028 running (5m) 28s ago 17m 414M 1937M 19.3.0-4917-g8293d73f 425e316ba8eb df34bd603d8f 2024-09-16T08:25:46.166 INFO:teuthology.orchestra.run.smithi028.stdout:osd.2 smithi028 running (4m) 28s ago 16m 308M 1937M 19.3.0-4917-g8293d73f 425e316ba8eb c92be6327b12 2024-09-16T08:25:46.166 INFO:teuthology.orchestra.run.smithi028.stdout:osd.3 smithi028 running (3m) 28s ago 16m 399M 1937M 19.3.0-4917-g8293d73f 425e316ba8eb bd84ba1224f2 2024-09-16T08:25:46.166 INFO:teuthology.orchestra.run.smithi028.stdout:osd.4 smithi144 running (2m) 9s ago 16m 339M 2704M 19.3.0-4917-g8293d73f 425e316ba8eb c48cb68e901e 2024-09-16T08:25:46.166 INFO:teuthology.orchestra.run.smithi028.stdout:osd.5 smithi144 running (2m) 9s ago 15m 335M 2704M 19.3.0-4917-g8293d73f 425e316ba8eb 59ec97fb3cf3 2024-09-16T08:25:46.166 INFO:teuthology.orchestra.run.smithi028.stdout:osd.6 smithi144 running (101s) 9s ago 15m 306M 2704M 19.3.0-4917-g8293d73f 425e316ba8eb b301bf0731f0 2024-09-16T08:25:46.166 INFO:teuthology.orchestra.run.smithi028.stdout:osd.7 smithi144 running (62s) 9s ago 14m 292M 2704M 19.3.0-4917-g8293d73f 425e316ba8eb f1a5cbf3a7bd 2024-09-16T08:25:46.166 INFO:teuthology.orchestra.run.smithi028.stdout:prometheus.smithi028 smithi028 *:9095 running (8m) 28s ago 18m 55.6M - 2.51.0 1d3b7f56885b 42667809df2b 2024-09-16T08:25:46.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:25:45 smithi144 ceph-mon[49149]: pgmap v326: 97 pgs: 97 active+clean; 232 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 2.3 KiB/s rd, 7.7 KiB/s wr, 5 op/s 2024-09-16T08:25:46.642 INFO:teuthology.orchestra.run.smithi028.stdout:{ 2024-09-16T08:25:46.642 INFO:teuthology.orchestra.run.smithi028.stdout: "mon": { 2024-09-16T08:25:46.642 INFO:teuthology.orchestra.run.smithi028.stdout: "ceph version 19.3.0-4917-g8293d73f (8293d73f8690540e843a81caec373f9cc29cf705) squid (dev)": 2 2024-09-16T08:25:46.642 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-09-16T08:25:46.643 INFO:teuthology.orchestra.run.smithi028.stdout: "mgr": { 2024-09-16T08:25:46.643 INFO:teuthology.orchestra.run.smithi028.stdout: "ceph version 19.3.0-4917-g8293d73f (8293d73f8690540e843a81caec373f9cc29cf705) squid (dev)": 2 2024-09-16T08:25:46.643 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-09-16T08:25:46.643 INFO:teuthology.orchestra.run.smithi028.stdout: "osd": { 2024-09-16T08:25:46.643 INFO:teuthology.orchestra.run.smithi028.stdout: "ceph version 19.3.0-4917-g8293d73f (8293d73f8690540e843a81caec373f9cc29cf705) squid (dev)": 8 2024-09-16T08:25:46.643 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-09-16T08:25:46.643 INFO:teuthology.orchestra.run.smithi028.stdout: "mds": { 2024-09-16T08:25:46.644 INFO:teuthology.orchestra.run.smithi028.stdout: "ceph version 19.3.0-4917-g8293d73f (8293d73f8690540e843a81caec373f9cc29cf705) squid (dev)": 2 2024-09-16T08:25:46.644 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-09-16T08:25:46.644 INFO:teuthology.orchestra.run.smithi028.stdout: "overall": { 2024-09-16T08:25:46.644 INFO:teuthology.orchestra.run.smithi028.stdout: "ceph version 19.3.0-4917-g8293d73f (8293d73f8690540e843a81caec373f9cc29cf705) squid (dev)": 14 2024-09-16T08:25:46.644 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-09-16T08:25:46.644 INFO:teuthology.orchestra.run.smithi028.stdout:} 2024-09-16T08:25:47.023 INFO:teuthology.orchestra.run.smithi028.stdout:{ 2024-09-16T08:25:47.023 INFO:teuthology.orchestra.run.smithi028.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:f7ad55f9ccd17ba66aadb19b4d291e67b60ea24e5d8a0c0e92941fd59c81e4ce", 2024-09-16T08:25:47.023 INFO:teuthology.orchestra.run.smithi028.stdout: "in_progress": true, 2024-09-16T08:25:47.023 INFO:teuthology.orchestra.run.smithi028.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-16T08:25:47.023 INFO:teuthology.orchestra.run.smithi028.stdout: "services_complete": [ 2024-09-16T08:25:47.023 INFO:teuthology.orchestra.run.smithi028.stdout: "mgr", 2024-09-16T08:25:47.023 INFO:teuthology.orchestra.run.smithi028.stdout: "crash", 2024-09-16T08:25:47.024 INFO:teuthology.orchestra.run.smithi028.stdout: "mon", 2024-09-16T08:25:47.024 INFO:teuthology.orchestra.run.smithi028.stdout: "osd", 2024-09-16T08:25:47.024 INFO:teuthology.orchestra.run.smithi028.stdout: "mds" 2024-09-16T08:25:47.024 INFO:teuthology.orchestra.run.smithi028.stdout: ], 2024-09-16T08:25:47.024 INFO:teuthology.orchestra.run.smithi028.stdout: "progress": "16/23 daemons upgraded", 2024-09-16T08:25:47.024 INFO:teuthology.orchestra.run.smithi028.stdout: "message": "Currently upgrading nfs daemons", 2024-09-16T08:25:47.024 INFO:teuthology.orchestra.run.smithi028.stdout: "is_paused": false 2024-09-16T08:25:47.024 INFO:teuthology.orchestra.run.smithi028.stdout:} 2024-09-16T08:25:47.120 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:25:46 smithi144 ceph-mon[49149]: from='client.34500 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T08:25:47.121 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:25:46 smithi144 ceph-mon[49149]: from='client.34504 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T08:25:47.121 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:25:46 smithi144 ceph-mon[49149]: from='client.34508 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T08:25:47.121 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:25:46 smithi144 ceph-mon[49149]: from='mds.? [v2:172.21.15.28:6834/1804746814,v1:172.21.15.28:6835/1804746814]' entity='mds.foofs.smithi028.srtgev' cmd={"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.28:0/1295173080"} : dispatch 2024-09-16T08:25:47.121 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:25:46 smithi144 ceph-mon[49149]: from='mds.? ' entity='mds.foofs.smithi028.srtgev' cmd={"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.28:0/1295173080"} : dispatch 2024-09-16T08:25:47.121 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:25:46 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:25:47.121 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:25:46 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:25:47.121 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:25:46 smithi144 ceph-mon[49149]: from='client.? 172.21.15.28:0/3471360907' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-16T08:25:47.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:25:46 smithi028 ceph-mon[64115]: from='client.34500 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T08:25:47.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:25:46 smithi028 ceph-mon[64115]: from='client.34504 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T08:25:47.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:25:46 smithi028 ceph-mon[64115]: from='client.34508 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T08:25:47.448 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:25:46 smithi028 ceph-mon[64115]: from='mds.? [v2:172.21.15.28:6834/1804746814,v1:172.21.15.28:6835/1804746814]' entity='mds.foofs.smithi028.srtgev' cmd={"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.28:0/1295173080"} : dispatch 2024-09-16T08:25:47.448 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:25:46 smithi028 ceph-mon[64115]: from='mds.? ' entity='mds.foofs.smithi028.srtgev' cmd={"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.28:0/1295173080"} : dispatch 2024-09-16T08:25:47.448 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:25:46 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:25:47.448 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:25:46 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:25:47.448 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:25:46 smithi028 ceph-mon[64115]: from='client.? 172.21.15.28:0/3471360907' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-16T08:25:47.460 INFO:teuthology.orchestra.run.smithi028.stdout:HEALTH_OK 2024-09-16T08:25:48.306 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:25:48 smithi144 ceph-mon[49149]: Evicting (and blocklisting) client session 14546 (172.21.15.28:0/1295173080) 2024-09-16T08:25:48.306 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:25:48 smithi144 ceph-mon[49149]: pgmap v327: 97 pgs: 97 active+clean; 232 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 1.2 KiB/s rd, 1023 B/s wr, 1 op/s 2024-09-16T08:25:48.306 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:25:48 smithi144 ceph-mon[49149]: from='mds.? ' entity='mds.foofs.smithi028.srtgev' cmd='[{"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.28:0/1295173080"}]': finished 2024-09-16T08:25:48.306 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:25:48 smithi144 ceph-mon[49149]: osdmap e157: 8 total, 8 up, 8 in 2024-09-16T08:25:48.306 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:25:48 smithi144 ceph-mon[49149]: from='client.34528 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T08:25:48.307 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:25:48 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:25:48.307 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:25:48 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-16T08:25:48.307 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:25:48 smithi144 ceph-mon[49149]: from='client.? 172.21.15.28:0/3679630215' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-16T08:25:48.307 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:25:48 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:25:48.307 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:25:48 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi144.epipkg", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]} : dispatch 2024-09-16T08:25:48.307 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:25:48 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]} : dispatch 2024-09-16T08:25:48.307 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:25:48 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-09-16T08:25:48.307 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:25:48 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-16T08:25:48.307 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:25:48 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"} : dispatch 2024-09-16T08:25:48.307 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:25:48 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-09-16T08:25:48.307 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:25:48 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi144.epipkg-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]} : dispatch 2024-09-16T08:25:48.308 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:25:48 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-16T08:25:48.448 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:25:48 smithi028 ceph-mon[64115]: Evicting (and blocklisting) client session 14546 (172.21.15.28:0/1295173080) 2024-09-16T08:25:48.448 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:25:48 smithi028 ceph-mon[64115]: pgmap v327: 97 pgs: 97 active+clean; 232 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 1.2 KiB/s rd, 1023 B/s wr, 1 op/s 2024-09-16T08:25:48.448 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:25:48 smithi028 ceph-mon[64115]: from='mds.? ' entity='mds.foofs.smithi028.srtgev' cmd='[{"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.28:0/1295173080"}]': finished 2024-09-16T08:25:48.448 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:25:48 smithi028 ceph-mon[64115]: osdmap e157: 8 total, 8 up, 8 in 2024-09-16T08:25:48.448 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:25:48 smithi028 ceph-mon[64115]: from='client.34528 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T08:25:48.449 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:25:48 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:25:48.449 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:25:48 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-16T08:25:48.449 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:25:48 smithi028 ceph-mon[64115]: from='client.? 172.21.15.28:0/3679630215' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-16T08:25:48.449 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:25:48 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:25:48.449 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:25:48 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi144.epipkg", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]} : dispatch 2024-09-16T08:25:48.449 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:25:48 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]} : dispatch 2024-09-16T08:25:48.449 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:25:48 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-09-16T08:25:48.449 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:25:48 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-16T08:25:48.449 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:25:48 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"} : dispatch 2024-09-16T08:25:48.449 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:25:48 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-09-16T08:25:48.449 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:25:48 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi144.epipkg-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]} : dispatch 2024-09-16T08:25:48.450 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:25:48 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-16T08:25:49.248 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:25:49 smithi144 ceph-mon[49149]: Upgrade: Updating nfs.foo.1.0.smithi144.epipkg (2/2) 2024-09-16T08:25:49.248 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:25:49 smithi144 ceph-mon[49149]: Creating key for client.nfs.foo.1.0.smithi144.epipkg 2024-09-16T08:25:49.248 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:25:49 smithi144 ceph-mon[49149]: Ensuring nfs.foo.1 is in the ganesha grace table 2024-09-16T08:25:49.248 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:25:49 smithi144 ceph-mon[49149]: Rados config object exists: conf-nfs.foo 2024-09-16T08:25:49.248 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:25:49 smithi144 ceph-mon[49149]: Creating key for client.nfs.foo.1.0.smithi144.epipkg-rgw 2024-09-16T08:25:49.249 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:25:49 smithi144 ceph-mon[49149]: Bind address in nfs.foo.1.0.smithi144.epipkg's ganesha conf is defaulting to empty 2024-09-16T08:25:49.249 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:25:49 smithi144 ceph-mon[49149]: Deploying daemon nfs.foo.1.0.smithi144.epipkg on smithi144 2024-09-16T08:25:49.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:25:49 smithi028 ceph-mon[64115]: Upgrade: Updating nfs.foo.1.0.smithi144.epipkg (2/2) 2024-09-16T08:25:49.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:25:49 smithi028 ceph-mon[64115]: Creating key for client.nfs.foo.1.0.smithi144.epipkg 2024-09-16T08:25:49.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:25:49 smithi028 ceph-mon[64115]: Ensuring nfs.foo.1 is in the ganesha grace table 2024-09-16T08:25:49.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:25:49 smithi028 ceph-mon[64115]: Rados config object exists: conf-nfs.foo 2024-09-16T08:25:49.448 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:25:49 smithi028 ceph-mon[64115]: Creating key for client.nfs.foo.1.0.smithi144.epipkg-rgw 2024-09-16T08:25:49.448 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:25:49 smithi028 ceph-mon[64115]: Bind address in nfs.foo.1.0.smithi144.epipkg's ganesha conf is defaulting to empty 2024-09-16T08:25:49.448 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:25:49 smithi028 ceph-mon[64115]: Deploying daemon nfs.foo.1.0.smithi144.epipkg on smithi144 2024-09-16T08:25:50.082 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:25:50 smithi144 ceph-mon[49149]: pgmap v329: 97 pgs: 97 active+clean; 232 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 1.5 KiB/s rd, 1.3 KiB/s wr, 2 op/s 2024-09-16T08:25:50.197 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:25:50 smithi028 ceph-mon[64115]: pgmap v329: 97 pgs: 97 active+clean; 232 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 1.5 KiB/s rd, 1.3 KiB/s wr, 2 op/s 2024-09-16T08:25:52.294 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:25:52 smithi144 ceph-mon[49149]: pgmap v330: 97 pgs: 97 active+clean; 232 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 2.9 KiB/s rd, 1.4 KiB/s wr, 4 op/s 2024-09-16T08:25:52.294 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:25:52 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:25:52.295 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:25:52 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:25:52.295 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:25:52 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-16T08:25:52.319 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:25:52 smithi028 ceph-mon[64115]: pgmap v330: 97 pgs: 97 active+clean; 232 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 2.9 KiB/s rd, 1.4 KiB/s wr, 4 op/s 2024-09-16T08:25:52.319 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:25:52 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:25:52.320 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:25:52 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:25:52.320 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:25:52 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-16T08:25:54.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:25:54 smithi144 ceph-mon[49149]: pgmap v331: 97 pgs: 97 active+clean; 232 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 3.0 KiB/s rd, 1.4 KiB/s wr, 4 op/s 2024-09-16T08:25:54.338 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:25:54 smithi028 ceph-mon[64115]: pgmap v331: 97 pgs: 97 active+clean; 232 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 3.0 KiB/s rd, 1.4 KiB/s wr, 4 op/s 2024-09-16T08:25:55.430 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:25:55 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:25:55.430 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:25:55 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:25:55.430 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:25:55 smithi144 ceph-mon[49149]: from='mds.? [v2:172.21.15.28:6834/1804746814,v1:172.21.15.28:6835/1804746814]' entity='mds.foofs.smithi028.srtgev' cmd={"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.144:0/2699919051"} : dispatch 2024-09-16T08:25:55.430 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:25:55 smithi144 ceph-mon[49149]: from='mds.? ' entity='mds.foofs.smithi028.srtgev' cmd={"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.144:0/2699919051"} : dispatch 2024-09-16T08:25:55.431 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:25:55 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:25:55.431 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:25:55 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:25:55.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:25:55 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:25:55.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:25:55 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:25:55.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:25:55 smithi028 ceph-mon[64115]: from='mds.? [v2:172.21.15.28:6834/1804746814,v1:172.21.15.28:6835/1804746814]' entity='mds.foofs.smithi028.srtgev' cmd={"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.144:0/2699919051"} : dispatch 2024-09-16T08:25:55.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:25:55 smithi028 ceph-mon[64115]: from='mds.? ' entity='mds.foofs.smithi028.srtgev' cmd={"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.144:0/2699919051"} : dispatch 2024-09-16T08:25:55.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:25:55 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:25:55.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:25:55 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:25:56.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:25:56 smithi144 ceph-mon[49149]: Evicting (and blocklisting) client session 24359 (172.21.15.144:0/2699919051) 2024-09-16T08:25:56.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:25:56 smithi144 ceph-mon[49149]: pgmap v332: 97 pgs: 97 active+clean; 232 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 3.8 KiB/s rd, 1.8 KiB/s wr, 6 op/s 2024-09-16T08:25:56.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:25:56 smithi144 ceph-mon[49149]: from='mds.? ' entity='mds.foofs.smithi028.srtgev' cmd='[{"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.144:0/2699919051"}]': finished 2024-09-16T08:25:56.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:25:56 smithi144 ceph-mon[49149]: osdmap e158: 8 total, 8 up, 8 in 2024-09-16T08:25:56.449 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:25:56 smithi028 ceph-mon[64115]: Evicting (and blocklisting) client session 24359 (172.21.15.144:0/2699919051) 2024-09-16T08:25:56.449 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:25:56 smithi028 ceph-mon[64115]: pgmap v332: 97 pgs: 97 active+clean; 232 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 3.8 KiB/s rd, 1.8 KiB/s wr, 6 op/s 2024-09-16T08:25:56.449 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:25:56 smithi028 ceph-mon[64115]: from='mds.? ' entity='mds.foofs.smithi028.srtgev' cmd='[{"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.144:0/2699919051"}]': finished 2024-09-16T08:25:56.449 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:25:56 smithi028 ceph-mon[64115]: osdmap e158: 8 total, 8 up, 8 in 2024-09-16T08:25:57.451 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:25:57 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:25:57.451 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:25:57 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:25:57.451 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:25:57 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:25:57.451 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:25:57 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:25:57.467 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:25:57 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:25:57.467 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:25:57 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:25:57.467 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:25:57 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:25:57.467 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:25:57 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:25:58.472 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:25:58 smithi028 ceph-mon[64115]: pgmap v334: 97 pgs: 97 active+clean; 232 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 3.6 KiB/s rd, 1.7 KiB/s wr, 6 op/s 2024-09-16T08:25:58.472 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:25:58 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:25:58.472 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:25:58 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:25:58.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:25:58 smithi144 ceph-mon[49149]: pgmap v334: 97 pgs: 97 active+clean; 232 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 3.6 KiB/s rd, 1.7 KiB/s wr, 6 op/s 2024-09-16T08:25:58.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:25:58 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:25:58.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:25:58 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:26:00.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:26:00 smithi144 ceph-mon[49149]: pgmap v335: 97 pgs: 97 active+clean; 232 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 3.6 KiB/s rd, 1.6 KiB/s wr, 5 op/s 2024-09-16T08:26:00.697 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:26:00 smithi028 ceph-mon[64115]: pgmap v335: 97 pgs: 97 active+clean; 232 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 3.6 KiB/s rd, 1.6 KiB/s wr, 5 op/s 2024-09-16T08:26:01.432 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:26:01 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:26:01.433 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:26:01 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:26:01.433 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:26:01 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-16T08:26:01.433 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:26:01 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-16T08:26:01.433 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:26:01 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:26:01.433 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:26:01 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:26:01.433 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:26:01 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-16T08:26:01.433 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:26:01 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "versions"} : dispatch 2024-09-16T08:26:01.433 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:26:01 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "versions"} : dispatch 2024-09-16T08:26:01.433 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:26:01 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "versions"} : dispatch 2024-09-16T08:26:01.433 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:26:01 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "versions"} : dispatch 2024-09-16T08:26:01.433 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:26:01 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "versions"} : dispatch 2024-09-16T08:26:01.434 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:26:01 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "versions"} : dispatch 2024-09-16T08:26:01.434 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:26:01 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "versions"} : dispatch 2024-09-16T08:26:01.434 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:26:01 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "versions"} : dispatch 2024-09-16T08:26:01.434 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:26:01 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "versions"} : dispatch 2024-09-16T08:26:01.434 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:26:01 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "versions"} : dispatch 2024-09-16T08:26:01.434 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:26:01 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "versions"} : dispatch 2024-09-16T08:26:01.434 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:26:01 smithi028 ceph-mon[64115]: Upgrade: Setting container_image for all nfs 2024-09-16T08:26:01.434 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:26:01 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:26:01.434 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:26:01 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.0.0.smithi028.dkttgj"} : dispatch 2024-09-16T08:26:01.434 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:26:01 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.0.0.smithi028.dkttgj"}]': finished 2024-09-16T08:26:01.434 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:26:01 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.1.0.smithi144.epipkg"} : dispatch 2024-09-16T08:26:01.435 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:26:01 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.1.0.smithi144.epipkg"}]': finished 2024-09-16T08:26:01.435 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:26:01 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "versions"} : dispatch 2024-09-16T08:26:01.435 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:26:01 smithi028 ceph-mon[64115]: Upgrade: Setting container_image for all nvmeof 2024-09-16T08:26:01.435 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:26:01 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:26:01.435 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:26:01 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "versions"} : dispatch 2024-09-16T08:26:01.435 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:26:01 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "versions"} : dispatch 2024-09-16T08:26:01.435 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:26:01 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "versions"} : dispatch 2024-09-16T08:26:01.435 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:26:01 smithi028 ceph-mon[64115]: pgmap v336: 97 pgs: 97 active+clean; 232 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 2.1 KiB/s rd, 1.2 KiB/s wr, 3 op/s 2024-09-16T08:26:01.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:26:01 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:26:01.583 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:26:01 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:26:01.583 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:26:01 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-16T08:26:01.583 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:26:01 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-16T08:26:01.583 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:26:01 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:26:01.583 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:26:01 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:26:01.583 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:26:01 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-16T08:26:01.583 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:26:01 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "versions"} : dispatch 2024-09-16T08:26:01.583 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:26:01 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "versions"} : dispatch 2024-09-16T08:26:01.583 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:26:01 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "versions"} : dispatch 2024-09-16T08:26:01.583 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:26:01 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "versions"} : dispatch 2024-09-16T08:26:01.583 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:26:01 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "versions"} : dispatch 2024-09-16T08:26:01.583 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:26:01 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "versions"} : dispatch 2024-09-16T08:26:01.583 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:26:01 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "versions"} : dispatch 2024-09-16T08:26:01.584 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:26:01 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "versions"} : dispatch 2024-09-16T08:26:01.584 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:26:01 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "versions"} : dispatch 2024-09-16T08:26:01.584 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:26:01 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "versions"} : dispatch 2024-09-16T08:26:01.584 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:26:01 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "versions"} : dispatch 2024-09-16T08:26:01.584 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:26:01 smithi144 ceph-mon[49149]: Upgrade: Setting container_image for all nfs 2024-09-16T08:26:01.584 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:26:01 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:26:01.584 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:26:01 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.0.0.smithi028.dkttgj"} : dispatch 2024-09-16T08:26:01.584 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:26:01 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.0.0.smithi028.dkttgj"}]': finished 2024-09-16T08:26:01.584 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:26:01 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.1.0.smithi144.epipkg"} : dispatch 2024-09-16T08:26:01.584 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:26:01 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.1.0.smithi144.epipkg"}]': finished 2024-09-16T08:26:01.584 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:26:01 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "versions"} : dispatch 2024-09-16T08:26:01.584 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:26:01 smithi144 ceph-mon[49149]: Upgrade: Setting container_image for all nvmeof 2024-09-16T08:26:01.585 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:26:01 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:26:01.585 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:26:01 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "versions"} : dispatch 2024-09-16T08:26:01.585 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:26:01 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "versions"} : dispatch 2024-09-16T08:26:01.585 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:26:01 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "versions"} : dispatch 2024-09-16T08:26:01.585 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:26:01 smithi144 ceph-mon[49149]: pgmap v336: 97 pgs: 97 active+clean; 232 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 2.1 KiB/s rd, 1.2 KiB/s wr, 3 op/s 2024-09-16T08:26:02.802 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:26:02 smithi028 ceph-mon[64115]: Upgrade: Updating grafana.smithi028 2024-09-16T08:26:02.802 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:26:02 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-16T08:26:03.082 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:26:02 smithi144 ceph-mon[49149]: Upgrade: Updating grafana.smithi028 2024-09-16T08:26:03.082 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:26:02 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-16T08:26:04.082 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:26:03 smithi144 ceph-mon[49149]: Deploying daemon grafana.smithi028 on smithi028 2024-09-16T08:26:04.082 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:26:03 smithi144 ceph-mon[49149]: pgmap v337: 97 pgs: 97 active+clean; 232 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 2.0 KiB/s rd, 1.2 KiB/s wr, 3 op/s 2024-09-16T08:26:04.197 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:26:03 smithi028 ceph-mon[64115]: Deploying daemon grafana.smithi028 on smithi028 2024-09-16T08:26:04.197 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:26:03 smithi028 ceph-mon[64115]: pgmap v337: 97 pgs: 97 active+clean; 232 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 2.0 KiB/s rd, 1.2 KiB/s wr, 3 op/s 2024-09-16T08:26:06.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:26:05 smithi144 ceph-mon[49149]: pgmap v338: 97 pgs: 97 active+clean; 232 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 204 B/s rd, 307 B/s wr, 0 op/s 2024-09-16T08:26:06.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:26:05 smithi028 ceph-mon[64115]: pgmap v338: 97 pgs: 97 active+clean; 232 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 204 B/s rd, 307 B/s wr, 0 op/s 2024-09-16T08:26:08.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:26:08 smithi144 ceph-mon[49149]: pgmap v339: 97 pgs: 97 active+clean; 232 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 173 B/s rd, 259 B/s wr, 0 op/s 2024-09-16T08:26:08.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:26:07 smithi028 ceph-mon[64115]: pgmap v339: 97 pgs: 97 active+clean; 232 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 173 B/s rd, 259 B/s wr, 0 op/s 2024-09-16T08:26:10.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:26:09 smithi144 ceph-mon[49149]: pgmap v340: 97 pgs: 97 active+clean; 232 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 255 B/s wr, 0 op/s 2024-09-16T08:26:10.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:26:09 smithi028 ceph-mon[64115]: pgmap v340: 97 pgs: 97 active+clean; 232 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 255 B/s wr, 0 op/s 2024-09-16T08:26:12.197 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:26:11 smithi028 ceph-mon[64115]: pgmap v341: 97 pgs: 97 active+clean; 232 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 255 B/s wr, 0 op/s 2024-09-16T08:26:12.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:26:11 smithi144 ceph-mon[49149]: pgmap v341: 97 pgs: 97 active+clean; 232 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 255 B/s wr, 0 op/s 2024-09-16T08:26:14.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:26:14 smithi144 ceph-mon[49149]: pgmap v342: 97 pgs: 97 active+clean; 232 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-16T08:26:14.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:26:14 smithi028 ceph-mon[64115]: pgmap v342: 97 pgs: 97 active+clean; 232 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-16T08:26:16.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:26:16 smithi144 ceph-mon[49149]: pgmap v343: 97 pgs: 97 active+clean; 232 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-09-16T08:26:16.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:26:16 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:26:16.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:26:16 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:26:16.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:26:16 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-16T08:26:16.363 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:26:16 smithi028 ceph-mon[64115]: pgmap v343: 97 pgs: 97 active+clean; 232 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-09-16T08:26:16.363 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:26:16 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:26:16.363 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:26:16 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:26:16.363 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:26:16 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-16T08:26:17.853 INFO:teuthology.orchestra.run.smithi028.stdout:true 2024-09-16T08:26:18.197 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:26:18 smithi028 ceph-mon[64115]: pgmap v344: 97 pgs: 97 active+clean; 232 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-16T08:26:18.197 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:26:18 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-16T08:26:18.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:26:18 smithi144 ceph-mon[49149]: pgmap v344: 97 pgs: 97 active+clean; 232 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-16T08:26:18.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:26:18 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-16T08:26:18.613 INFO:teuthology.orchestra.run.smithi028.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-16T08:26:18.613 INFO:teuthology.orchestra.run.smithi028.stdout:alertmanager.smithi028 smithi028 *:9093,9094 running (10m) 22s ago 19m 22.2M - 0.27.0 11f11916f8cd a14fcb4cb2b6 2024-09-16T08:26:18.613 INFO:teuthology.orchestra.run.smithi028.stdout:crash.smithi028 smithi028 running (7m) 22s ago 19m 6899k - 19.3.0-4917-g8293d73f 425e316ba8eb 4bb89df2a867 2024-09-16T08:26:18.614 INFO:teuthology.orchestra.run.smithi028.stdout:crash.smithi144 smithi144 running (7m) 24s ago 19m 6895k - 19.3.0-4917-g8293d73f 425e316ba8eb 24a96875d7af 2024-09-16T08:26:18.614 INFO:teuthology.orchestra.run.smithi028.stdout:grafana.smithi028 smithi028 *:3000 starting - - - - 2024-09-16T08:26:18.614 INFO:teuthology.orchestra.run.smithi028.stdout:mds.foofs.smithi028.srtgev smithi028 running (67s) 22s ago 13m 32.5M - 19.3.0-4917-g8293d73f 425e316ba8eb 102d36a11737 2024-09-16T08:26:18.614 INFO:teuthology.orchestra.run.smithi028.stdout:mds.foofs.smithi144.oozgsi smithi144 running (44s) 24s ago 13m 13.0M - 19.3.0-4917-g8293d73f 425e316ba8eb 9917dfd27777 2024-09-16T08:26:18.614 INFO:teuthology.orchestra.run.smithi028.stdout:mgr.smithi028.azvyzy smithi028 *:8443,9283,8765 running (9m) 22s ago 20m 524M - 19.3.0-4917-g8293d73f 425e316ba8eb e01d4382d03f 2024-09-16T08:26:18.614 INFO:teuthology.orchestra.run.smithi028.stdout:mgr.smithi144.utzszj smithi144 *:8443,9283,8765 running (8m) 24s ago 19m 458M - 19.3.0-4917-g8293d73f 425e316ba8eb 71f7f802bffb 2024-09-16T08:26:18.614 INFO:teuthology.orchestra.run.smithi028.stdout:mon.smithi028 smithi028 running (8m) 22s ago 20m 62.8M 2048M 19.3.0-4917-g8293d73f 425e316ba8eb 14efcedc0088 2024-09-16T08:26:18.614 INFO:teuthology.orchestra.run.smithi028.stdout:mon.smithi144 smithi144 running (8m) 24s ago 18m 51.3M 2048M 19.3.0-4917-g8293d73f 425e316ba8eb c081c5ed44b5 2024-09-16T08:26:18.614 INFO:teuthology.orchestra.run.smithi028.stdout:nfs.foo.0.0.smithi028.dkttgj smithi028 *:2049 running (32s) 22s ago 13m 62.1M - 5.9 425e316ba8eb 8a211930c928 2024-09-16T08:26:18.614 INFO:teuthology.orchestra.run.smithi028.stdout:nfs.foo.1.0.smithi144.epipkg smithi144 *:2049 running (27s) 24s ago 13m 19.8M - 5.9 425e316ba8eb 1a56ade0da3e 2024-09-16T08:26:18.614 INFO:teuthology.orchestra.run.smithi028.stdout:node-exporter.smithi028 smithi028 *:9100 running (10m) 22s ago 19m 11.5M - 1.7.0 72c9c2088986 9e2b8e8a63a9 2024-09-16T08:26:18.614 INFO:teuthology.orchestra.run.smithi028.stdout:node-exporter.smithi144 smithi144 *:9100 running (9m) 24s ago 18m 10.2M - 1.7.0 72c9c2088986 78834c5e992f 2024-09-16T08:26:18.615 INFO:teuthology.orchestra.run.smithi028.stdout:osd.0 smithi028 running (6m) 22s ago 18m 428M 1937M 19.3.0-4917-g8293d73f 425e316ba8eb 2fc02ba4f14f 2024-09-16T08:26:18.615 INFO:teuthology.orchestra.run.smithi028.stdout:osd.1 smithi028 running (5m) 22s ago 17m 416M 1937M 19.3.0-4917-g8293d73f 425e316ba8eb df34bd603d8f 2024-09-16T08:26:18.615 INFO:teuthology.orchestra.run.smithi028.stdout:osd.2 smithi028 running (5m) 22s ago 17m 308M 1937M 19.3.0-4917-g8293d73f 425e316ba8eb c92be6327b12 2024-09-16T08:26:18.615 INFO:teuthology.orchestra.run.smithi028.stdout:osd.3 smithi028 running (4m) 22s ago 16m 417M 1937M 19.3.0-4917-g8293d73f 425e316ba8eb bd84ba1224f2 2024-09-16T08:26:18.615 INFO:teuthology.orchestra.run.smithi028.stdout:osd.4 smithi144 running (3m) 24s ago 16m 340M 2704M 19.3.0-4917-g8293d73f 425e316ba8eb c48cb68e901e 2024-09-16T08:26:18.615 INFO:teuthology.orchestra.run.smithi028.stdout:osd.5 smithi144 running (2m) 24s ago 16m 335M 2704M 19.3.0-4917-g8293d73f 425e316ba8eb 59ec97fb3cf3 2024-09-16T08:26:18.615 INFO:teuthology.orchestra.run.smithi028.stdout:osd.6 smithi144 running (2m) 24s ago 15m 307M 2704M 19.3.0-4917-g8293d73f 425e316ba8eb b301bf0731f0 2024-09-16T08:26:18.615 INFO:teuthology.orchestra.run.smithi028.stdout:osd.7 smithi144 running (95s) 24s ago 15m 292M 2704M 19.3.0-4917-g8293d73f 425e316ba8eb f1a5cbf3a7bd 2024-09-16T08:26:18.615 INFO:teuthology.orchestra.run.smithi028.stdout:prometheus.smithi028 smithi028 *:9095 running (9m) 22s ago 18m 56.3M - 2.51.0 1d3b7f56885b 42667809df2b 2024-09-16T08:26:19.082 INFO:teuthology.orchestra.run.smithi028.stdout:{ 2024-09-16T08:26:19.083 INFO:teuthology.orchestra.run.smithi028.stdout: "mon": { 2024-09-16T08:26:19.083 INFO:teuthology.orchestra.run.smithi028.stdout: "ceph version 19.3.0-4917-g8293d73f (8293d73f8690540e843a81caec373f9cc29cf705) squid (dev)": 2 2024-09-16T08:26:19.083 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-09-16T08:26:19.083 INFO:teuthology.orchestra.run.smithi028.stdout: "mgr": { 2024-09-16T08:26:19.083 INFO:teuthology.orchestra.run.smithi028.stdout: "ceph version 19.3.0-4917-g8293d73f (8293d73f8690540e843a81caec373f9cc29cf705) squid (dev)": 2 2024-09-16T08:26:19.083 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-09-16T08:26:19.083 INFO:teuthology.orchestra.run.smithi028.stdout: "osd": { 2024-09-16T08:26:19.083 INFO:teuthology.orchestra.run.smithi028.stdout: "ceph version 19.3.0-4917-g8293d73f (8293d73f8690540e843a81caec373f9cc29cf705) squid (dev)": 8 2024-09-16T08:26:19.083 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-09-16T08:26:19.084 INFO:teuthology.orchestra.run.smithi028.stdout: "mds": { 2024-09-16T08:26:19.084 INFO:teuthology.orchestra.run.smithi028.stdout: "ceph version 19.3.0-4917-g8293d73f (8293d73f8690540e843a81caec373f9cc29cf705) squid (dev)": 2 2024-09-16T08:26:19.084 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-09-16T08:26:19.084 INFO:teuthology.orchestra.run.smithi028.stdout: "overall": { 2024-09-16T08:26:19.084 INFO:teuthology.orchestra.run.smithi028.stdout: "ceph version 19.3.0-4917-g8293d73f (8293d73f8690540e843a81caec373f9cc29cf705) squid (dev)": 14 2024-09-16T08:26:19.084 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-09-16T08:26:19.084 INFO:teuthology.orchestra.run.smithi028.stdout:} 2024-09-16T08:26:19.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:26:19 smithi028 ceph-mon[64115]: from='client.34548 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T08:26:19.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:26:19 smithi028 ceph-mon[64115]: from='client.34552 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T08:26:19.453 INFO:teuthology.orchestra.run.smithi028.stdout:{ 2024-09-16T08:26:19.453 INFO:teuthology.orchestra.run.smithi028.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:f7ad55f9ccd17ba66aadb19b4d291e67b60ea24e5d8a0c0e92941fd59c81e4ce", 2024-09-16T08:26:19.453 INFO:teuthology.orchestra.run.smithi028.stdout: "in_progress": true, 2024-09-16T08:26:19.453 INFO:teuthology.orchestra.run.smithi028.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-16T08:26:19.453 INFO:teuthology.orchestra.run.smithi028.stdout: "services_complete": [ 2024-09-16T08:26:19.453 INFO:teuthology.orchestra.run.smithi028.stdout: "nfs", 2024-09-16T08:26:19.453 INFO:teuthology.orchestra.run.smithi028.stdout: "mgr", 2024-09-16T08:26:19.454 INFO:teuthology.orchestra.run.smithi028.stdout: "crash", 2024-09-16T08:26:19.454 INFO:teuthology.orchestra.run.smithi028.stdout: "mon", 2024-09-16T08:26:19.454 INFO:teuthology.orchestra.run.smithi028.stdout: "osd", 2024-09-16T08:26:19.454 INFO:teuthology.orchestra.run.smithi028.stdout: "mds" 2024-09-16T08:26:19.454 INFO:teuthology.orchestra.run.smithi028.stdout: ], 2024-09-16T08:26:19.454 INFO:teuthology.orchestra.run.smithi028.stdout: "progress": "18/23 daemons upgraded", 2024-09-16T08:26:19.454 INFO:teuthology.orchestra.run.smithi028.stdout: "message": "Currently upgrading grafana daemons", 2024-09-16T08:26:19.454 INFO:teuthology.orchestra.run.smithi028.stdout: "is_paused": false 2024-09-16T08:26:19.454 INFO:teuthology.orchestra.run.smithi028.stdout:} 2024-09-16T08:26:19.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:26:19 smithi144 ceph-mon[49149]: from='client.34548 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T08:26:19.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:26:19 smithi144 ceph-mon[49149]: from='client.34552 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T08:26:19.921 INFO:teuthology.orchestra.run.smithi028.stdout:HEALTH_OK 2024-09-16T08:26:20.197 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:26:20 smithi028 ceph-mon[64115]: from='client.34556 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T08:26:20.197 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:26:20 smithi028 ceph-mon[64115]: pgmap v345: 97 pgs: 97 active+clean; 232 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-16T08:26:20.197 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:26:20 smithi028 ceph-mon[64115]: from='client.? 172.21.15.28:0/3214372098' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-16T08:26:20.197 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:26:20 smithi028 ceph-mon[64115]: from='client.? 172.21.15.28:0/3385600719' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-16T08:26:20.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:26:20 smithi144 ceph-mon[49149]: from='client.34556 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T08:26:20.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:26:20 smithi144 ceph-mon[49149]: pgmap v345: 97 pgs: 97 active+clean; 232 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-16T08:26:20.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:26:20 smithi144 ceph-mon[49149]: from='client.? 172.21.15.28:0/3214372098' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-16T08:26:20.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:26:20 smithi144 ceph-mon[49149]: from='client.? 172.21.15.28:0/3385600719' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-16T08:26:21.414 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:26:21 smithi028 ceph-mon[64115]: from='client.34564 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T08:26:21.414 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:26:21 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:26:21.414 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:26:21 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:26:21.414 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:26:21 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:26:21.414 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:26:21 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:26:21.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:26:21 smithi144 ceph-mon[49149]: from='client.34564 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T08:26:21.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:26:21 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:26:21.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:26:21 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:26:21.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:26:21 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:26:21.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:26:21 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:26:22.369 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:26:22 smithi028 ceph-mon[64115]: pgmap v346: 97 pgs: 97 active+clean; 232 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-09-16T08:26:22.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:26:22 smithi144 ceph-mon[49149]: pgmap v346: 97 pgs: 97 active+clean; 232 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-09-16T08:26:24.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:26:24 smithi028 ceph-mon[64115]: pgmap v347: 97 pgs: 97 active+clean; 232 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-16T08:26:24.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:26:24 smithi144 ceph-mon[49149]: pgmap v347: 97 pgs: 97 active+clean; 232 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-16T08:26:26.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:26:26 smithi028 ceph-mon[64115]: pgmap v348: 97 pgs: 97 active+clean; 232 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-09-16T08:26:26.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:26:26 smithi144 ceph-mon[49149]: pgmap v348: 97 pgs: 97 active+clean; 232 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-09-16T08:26:27.948 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:26:27 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:26:27.948 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:26:27 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:26:27.948 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:26:27 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-16T08:26:27.948 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:26:27 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-16T08:26:27.948 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:26:27 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:26:27.948 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:26:27 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:26:27.948 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:26:27 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "dashboard get-grafana-api-url"} : dispatch 2024-09-16T08:26:27.948 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:26:27 smithi028 ceph-mon[64115]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-16T08:26:27.948 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:26:27 smithi028 ceph-mon[64115]: pgmap v349: 97 pgs: 97 active+clean; 232 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-16T08:26:27.948 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:26:27 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-16T08:26:27.948 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:26:27 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "versions"} : dispatch 2024-09-16T08:26:27.949 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:26:27 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "versions"} : dispatch 2024-09-16T08:26:27.949 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:26:27 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "versions"} : dispatch 2024-09-16T08:26:27.949 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:26:27 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "versions"} : dispatch 2024-09-16T08:26:27.949 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:26:27 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "versions"} : dispatch 2024-09-16T08:26:27.949 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:26:27 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "versions"} : dispatch 2024-09-16T08:26:27.949 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:26:27 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "versions"} : dispatch 2024-09-16T08:26:27.949 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:26:27 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "versions"} : dispatch 2024-09-16T08:26:27.949 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:26:27 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "versions"} : dispatch 2024-09-16T08:26:27.949 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:26:27 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "versions"} : dispatch 2024-09-16T08:26:27.949 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:26:27 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "versions"} : dispatch 2024-09-16T08:26:27.949 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:26:27 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "versions"} : dispatch 2024-09-16T08:26:27.949 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:26:27 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "versions"} : dispatch 2024-09-16T08:26:27.949 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:26:27 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "versions"} : dispatch 2024-09-16T08:26:27.950 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:26:27 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "versions"} : dispatch 2024-09-16T08:26:27.950 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:26:27 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "versions"} : dispatch 2024-09-16T08:26:27.950 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:26:27 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "versions"} : dispatch 2024-09-16T08:26:27.950 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:26:27 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "versions"} : dispatch 2024-09-16T08:26:27.950 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:26:27 smithi028 ceph-mon[64115]: Upgrade: Finalizing container_image settings 2024-09-16T08:26:27.950 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:26:27 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:26:27.950 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:26:27 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "config rm", "name": "container_image", "who": "mgr"} : dispatch 2024-09-16T08:26:27.950 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:26:27 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-09-16T08:26:27.950 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:26:27 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-16T08:26:27.950 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:26:27 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-09-16T08:26:27.950 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:26:27 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "config rm", "name": "container_image", "who": "client.crash"} : dispatch 2024-09-16T08:26:27.950 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:26:27 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-09-16T08:26:27.950 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:26:27 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "config rm", "name": "container_image", "who": "osd"} : dispatch 2024-09-16T08:26:27.951 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:26:27 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd"}]': finished 2024-09-16T08:26:27.951 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:26:27 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "config rm", "name": "container_image", "who": "mds"} : dispatch 2024-09-16T08:26:27.951 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:26:27 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-09-16T08:26:27.951 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:26:27 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rgw"} : dispatch 2024-09-16T08:26:27.951 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:26:27 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]': finished 2024-09-16T08:26:27.951 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:26:27 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"} : dispatch 2024-09-16T08:26:27.951 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:26:27 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-09-16T08:26:27.951 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:26:27 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-16T08:26:27.951 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:26:27 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"} : dispatch 2024-09-16T08:26:27.951 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:26:27 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-09-16T08:26:27.951 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:26:27 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "config rm", "name": "container_image", "who": "client.iscsi"} : dispatch 2024-09-16T08:26:27.952 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:26:27 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]': finished 2024-09-16T08:26:27.952 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:26:27 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nfs"} : dispatch 2024-09-16T08:26:27.952 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:26:27 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-09-16T08:26:27.952 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:26:27 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"} : dispatch 2024-09-16T08:26:27.952 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:26:27 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-09-16T08:26:27.952 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:26:27 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-16T08:26:27.952 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:26:27 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-16T08:26:27.952 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:26:27 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-16T08:26:27.952 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:26:27 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-16T08:26:27.953 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:26:27 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-16T08:26:27.953 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:26:27 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-16T08:26:27.953 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:26:27 smithi028 ceph-mon[64115]: Upgrade: Complete! 2024-09-16T08:26:27.953 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:26:27 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"} : dispatch 2024-09-16T08:26:27.953 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:26:27 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-09-16T08:26:27.953 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:26:27 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-16T08:26:27.953 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:26:27 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-16T08:26:27.953 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:26:27 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-16T08:26:27.953 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:26:27 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:26:27.953 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:26:27 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:26:27.954 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:26:27 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:26:27.954 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:26:27 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-16T08:26:27.954 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:26:27 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-16T08:26:27.954 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:26:27 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-16T08:26:28.082 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:26:27 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:26:28.083 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:26:27 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:26:28.083 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:26:27 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-16T08:26:28.083 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:26:27 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-16T08:26:28.083 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:26:27 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:26:28.083 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:26:27 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:26:28.083 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:26:27 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "dashboard get-grafana-api-url"} : dispatch 2024-09-16T08:26:28.083 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:26:27 smithi144 ceph-mon[49149]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-16T08:26:28.083 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:26:27 smithi144 ceph-mon[49149]: pgmap v349: 97 pgs: 97 active+clean; 232 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-16T08:26:28.083 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:26:27 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-16T08:26:28.083 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:26:27 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "versions"} : dispatch 2024-09-16T08:26:28.083 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:26:27 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "versions"} : dispatch 2024-09-16T08:26:28.084 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:26:27 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "versions"} : dispatch 2024-09-16T08:26:28.084 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:26:27 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "versions"} : dispatch 2024-09-16T08:26:28.084 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:26:27 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "versions"} : dispatch 2024-09-16T08:26:28.084 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:26:27 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "versions"} : dispatch 2024-09-16T08:26:28.084 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:26:27 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "versions"} : dispatch 2024-09-16T08:26:28.084 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:26:27 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "versions"} : dispatch 2024-09-16T08:26:28.084 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:26:27 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "versions"} : dispatch 2024-09-16T08:26:28.084 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:26:27 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "versions"} : dispatch 2024-09-16T08:26:28.084 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:26:27 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "versions"} : dispatch 2024-09-16T08:26:28.084 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:26:27 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "versions"} : dispatch 2024-09-16T08:26:28.084 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:26:27 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "versions"} : dispatch 2024-09-16T08:26:28.084 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:26:27 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "versions"} : dispatch 2024-09-16T08:26:28.084 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:26:27 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "versions"} : dispatch 2024-09-16T08:26:28.085 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:26:27 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "versions"} : dispatch 2024-09-16T08:26:28.085 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:26:27 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "versions"} : dispatch 2024-09-16T08:26:28.085 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:26:27 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "versions"} : dispatch 2024-09-16T08:26:28.085 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:26:27 smithi144 ceph-mon[49149]: Upgrade: Finalizing container_image settings 2024-09-16T08:26:28.085 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:26:27 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:26:28.085 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:26:27 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "config rm", "name": "container_image", "who": "mgr"} : dispatch 2024-09-16T08:26:28.085 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:26:27 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-09-16T08:26:28.086 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:26:27 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-16T08:26:28.086 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:26:27 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-09-16T08:26:28.086 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:26:27 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "config rm", "name": "container_image", "who": "client.crash"} : dispatch 2024-09-16T08:26:28.086 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:26:27 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-09-16T08:26:28.086 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:26:27 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "config rm", "name": "container_image", "who": "osd"} : dispatch 2024-09-16T08:26:28.086 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:26:27 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd"}]': finished 2024-09-16T08:26:28.086 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:26:27 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "config rm", "name": "container_image", "who": "mds"} : dispatch 2024-09-16T08:26:28.086 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:26:27 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-09-16T08:26:28.086 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:26:27 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rgw"} : dispatch 2024-09-16T08:26:28.086 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:26:27 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]': finished 2024-09-16T08:26:28.086 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:26:27 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"} : dispatch 2024-09-16T08:26:28.086 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:26:27 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-09-16T08:26:28.086 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:26:27 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-16T08:26:28.087 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:26:27 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"} : dispatch 2024-09-16T08:26:28.087 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:26:27 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-09-16T08:26:28.087 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:26:27 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "config rm", "name": "container_image", "who": "client.iscsi"} : dispatch 2024-09-16T08:26:28.087 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:26:27 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]': finished 2024-09-16T08:26:28.087 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:26:27 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nfs"} : dispatch 2024-09-16T08:26:28.087 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:26:27 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-09-16T08:26:28.087 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:26:27 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"} : dispatch 2024-09-16T08:26:28.087 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:26:27 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-09-16T08:26:28.087 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:26:27 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-16T08:26:28.087 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:26:27 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-16T08:26:28.087 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:26:27 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-16T08:26:28.087 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:26:27 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-16T08:26:28.087 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:26:27 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-16T08:26:28.087 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:26:27 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-16T08:26:28.088 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:26:27 smithi144 ceph-mon[49149]: Upgrade: Complete! 2024-09-16T08:26:28.088 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:26:27 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"} : dispatch 2024-09-16T08:26:28.088 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:26:27 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-09-16T08:26:28.088 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:26:27 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-16T08:26:28.088 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:26:27 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-16T08:26:28.088 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:26:27 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-16T08:26:28.088 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:26:27 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:26:28.088 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:26:27 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:26:28.088 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:26:27 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:26:28.088 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:26:27 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-16T08:26:28.088 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:26:27 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-16T08:26:28.088 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:26:27 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-16T08:26:28.947 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:26:28 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:26:28.947 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:26:28 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:26:29.082 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:26:28 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:26:29.082 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:26:28 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:26:29.947 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:26:29 smithi028 ceph-mon[64115]: pgmap v350: 97 pgs: 97 active+clean; 232 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-16T08:26:30.082 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:26:29 smithi144 ceph-mon[49149]: pgmap v350: 97 pgs: 97 active+clean; 232 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-16T08:26:32.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:26:31 smithi144 ceph-mon[49149]: pgmap v351: 97 pgs: 97 active+clean; 232 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-09-16T08:26:32.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:26:31 smithi028 ceph-mon[64115]: pgmap v351: 97 pgs: 97 active+clean; 232 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-09-16T08:26:33.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:26:33 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-16T08:26:33.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:26:33 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-16T08:26:34.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:26:34 smithi144 ceph-mon[49149]: pgmap v352: 97 pgs: 97 active+clean; 232 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-16T08:26:34.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:26:34 smithi028 ceph-mon[64115]: pgmap v352: 97 pgs: 97 active+clean; 232 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-16T08:26:36.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:26:36 smithi144 ceph-mon[49149]: pgmap v353: 97 pgs: 97 active+clean; 232 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-09-16T08:26:36.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:26:36 smithi028 ceph-mon[64115]: pgmap v353: 97 pgs: 97 active+clean; 232 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-09-16T08:26:38.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:26:38 smithi144 ceph-mon[49149]: pgmap v354: 97 pgs: 97 active+clean; 232 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-16T08:26:38.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:26:38 smithi028 ceph-mon[64115]: pgmap v354: 97 pgs: 97 active+clean; 232 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-16T08:26:40.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:26:40 smithi144 ceph-mon[49149]: pgmap v355: 97 pgs: 97 active+clean; 232 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-16T08:26:40.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:26:40 smithi028 ceph-mon[64115]: pgmap v355: 97 pgs: 97 active+clean; 232 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-16T08:26:42.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:26:42 smithi144 ceph-mon[49149]: pgmap v356: 97 pgs: 97 active+clean; 232 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-09-16T08:26:42.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:26:42 smithi028 ceph-mon[64115]: pgmap v356: 97 pgs: 97 active+clean; 232 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-09-16T08:26:44.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:26:44 smithi144 ceph-mon[49149]: pgmap v357: 97 pgs: 97 active+clean; 232 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-16T08:26:44.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:26:44 smithi028 ceph-mon[64115]: pgmap v357: 97 pgs: 97 active+clean; 232 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-16T08:26:46.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:26:46 smithi144 ceph-mon[49149]: pgmap v358: 97 pgs: 97 active+clean; 232 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-09-16T08:26:46.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:26:46 smithi028 ceph-mon[64115]: pgmap v358: 97 pgs: 97 active+clean; 232 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-09-16T08:26:48.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:26:48 smithi144 ceph-mon[49149]: pgmap v359: 97 pgs: 97 active+clean; 232 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-16T08:26:48.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:26:48 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-16T08:26:48.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:26:48 smithi028 ceph-mon[64115]: pgmap v359: 97 pgs: 97 active+clean; 232 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-16T08:26:48.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:26:48 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-16T08:26:50.303 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:26:50 smithi028 ceph-mon[64115]: pgmap v360: 97 pgs: 97 active+clean; 232 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-16T08:26:50.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:26:50 smithi144 ceph-mon[49149]: pgmap v360: 97 pgs: 97 active+clean; 232 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-16T08:26:51.082 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5622b2fe-7402-11ef-bceb-c7b262605968 -e sha1=8293d73f8690540e843a81caec373f9cc29cf705 -- bash -c 'ceph orch ps' 2024-09-16T08:26:52.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:26:52 smithi144 ceph-mon[49149]: from='client.34572 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T08:26:52.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:26:52 smithi144 ceph-mon[49149]: pgmap v361: 97 pgs: 97 active+clean; 232 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-09-16T08:26:52.363 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:26:52 smithi028 ceph-mon[64115]: from='client.34572 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T08:26:52.363 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:26:52 smithi028 ceph-mon[64115]: pgmap v361: 97 pgs: 97 active+clean; 232 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-09-16T08:26:53.101 INFO:teuthology.orchestra.run.smithi028.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-16T08:26:53.101 INFO:teuthology.orchestra.run.smithi028.stdout:alertmanager.smithi028 smithi028 *:9093,9094 running (11m) 32s ago 20m 22.2M - 0.27.0 11f11916f8cd a14fcb4cb2b6 2024-09-16T08:26:53.101 INFO:teuthology.orchestra.run.smithi028.stdout:crash.smithi028 smithi028 running (7m) 32s ago 20m 6899k - 19.3.0-4917-g8293d73f 425e316ba8eb 4bb89df2a867 2024-09-16T08:26:53.101 INFO:teuthology.orchestra.run.smithi028.stdout:crash.smithi144 smithi144 running (7m) 58s ago 19m 6895k - 19.3.0-4917-g8293d73f 425e316ba8eb 24a96875d7af 2024-09-16T08:26:53.101 INFO:teuthology.orchestra.run.smithi028.stdout:grafana.smithi028 smithi028 *:3000 running (38s) 32s ago 19m 38.7M - 10.4.0 c8b91775d855 79711ecc8811 2024-09-16T08:26:53.101 INFO:teuthology.orchestra.run.smithi028.stdout:mds.foofs.smithi028.srtgev smithi028 running (102s) 32s ago 14m 32.7M - 19.3.0-4917-g8293d73f 425e316ba8eb 102d36a11737 2024-09-16T08:26:53.101 INFO:teuthology.orchestra.run.smithi028.stdout:mds.foofs.smithi144.oozgsi smithi144 running (79s) 58s ago 14m 13.0M - 19.3.0-4917-g8293d73f 425e316ba8eb 9917dfd27777 2024-09-16T08:26:53.102 INFO:teuthology.orchestra.run.smithi028.stdout:mgr.smithi028.azvyzy smithi028 *:8443,9283,8765 running (10m) 32s ago 21m 533M - 19.3.0-4917-g8293d73f 425e316ba8eb e01d4382d03f 2024-09-16T08:26:53.102 INFO:teuthology.orchestra.run.smithi028.stdout:mgr.smithi144.utzszj smithi144 *:8443,9283,8765 running (9m) 58s ago 19m 458M - 19.3.0-4917-g8293d73f 425e316ba8eb 71f7f802bffb 2024-09-16T08:26:53.102 INFO:teuthology.orchestra.run.smithi028.stdout:mon.smithi028 smithi028 running (9m) 32s ago 21m 63.6M 2048M 19.3.0-4917-g8293d73f 425e316ba8eb 14efcedc0088 2024-09-16T08:26:53.102 INFO:teuthology.orchestra.run.smithi028.stdout:mon.smithi144 smithi144 running (8m) 58s ago 19m 51.3M 2048M 19.3.0-4917-g8293d73f 425e316ba8eb c081c5ed44b5 2024-09-16T08:26:53.102 INFO:teuthology.orchestra.run.smithi028.stdout:nfs.foo.0.0.smithi028.dkttgj smithi028 *:2049 running (67s) 32s ago 14m 63.0M - 5.9 425e316ba8eb 8a211930c928 2024-09-16T08:26:53.102 INFO:teuthology.orchestra.run.smithi028.stdout:nfs.foo.1.0.smithi144.epipkg smithi144 *:2049 running (62s) 58s ago 13m 19.8M - 5.9 425e316ba8eb 1a56ade0da3e 2024-09-16T08:26:53.102 INFO:teuthology.orchestra.run.smithi028.stdout:node-exporter.smithi028 smithi028 *:9100 running (10m) 32s ago 19m 11.7M - 1.7.0 72c9c2088986 9e2b8e8a63a9 2024-09-16T08:26:53.102 INFO:teuthology.orchestra.run.smithi028.stdout:node-exporter.smithi144 smithi144 *:9100 running (10m) 58s ago 19m 10.2M - 1.7.0 72c9c2088986 78834c5e992f 2024-09-16T08:26:53.102 INFO:teuthology.orchestra.run.smithi028.stdout:osd.0 smithi028 running (7m) 32s ago 18m 429M 1937M 19.3.0-4917-g8293d73f 425e316ba8eb 2fc02ba4f14f 2024-09-16T08:26:53.102 INFO:teuthology.orchestra.run.smithi028.stdout:osd.1 smithi028 running (6m) 32s ago 18m 417M 1937M 19.3.0-4917-g8293d73f 425e316ba8eb df34bd603d8f 2024-09-16T08:26:53.102 INFO:teuthology.orchestra.run.smithi028.stdout:osd.2 smithi028 running (5m) 32s ago 17m 309M 1937M 19.3.0-4917-g8293d73f 425e316ba8eb c92be6327b12 2024-09-16T08:26:53.102 INFO:teuthology.orchestra.run.smithi028.stdout:osd.3 smithi028 running (4m) 32s ago 17m 417M 1937M 19.3.0-4917-g8293d73f 425e316ba8eb bd84ba1224f2 2024-09-16T08:26:53.102 INFO:teuthology.orchestra.run.smithi028.stdout:osd.4 smithi144 running (4m) 58s ago 17m 340M 2704M 19.3.0-4917-g8293d73f 425e316ba8eb c48cb68e901e 2024-09-16T08:26:53.103 INFO:teuthology.orchestra.run.smithi028.stdout:osd.5 smithi144 running (3m) 58s ago 16m 335M 2704M 19.3.0-4917-g8293d73f 425e316ba8eb 59ec97fb3cf3 2024-09-16T08:26:53.103 INFO:teuthology.orchestra.run.smithi028.stdout:osd.6 smithi144 running (2m) 58s ago 16m 307M 2704M 19.3.0-4917-g8293d73f 425e316ba8eb b301bf0731f0 2024-09-16T08:26:53.103 INFO:teuthology.orchestra.run.smithi028.stdout:osd.7 smithi144 running (2m) 58s ago 15m 292M 2704M 19.3.0-4917-g8293d73f 425e316ba8eb f1a5cbf3a7bd 2024-09-16T08:26:53.103 INFO:teuthology.orchestra.run.smithi028.stdout:prometheus.smithi028 smithi028 *:9095 running (9m) 32s ago 19m 56.3M - 2.51.0 1d3b7f56885b 42667809df2b 2024-09-16T08:26:53.765 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5622b2fe-7402-11ef-bceb-c7b262605968 -e sha1=8293d73f8690540e843a81caec373f9cc29cf705 -- bash -c 'ceph versions' 2024-09-16T08:26:54.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:26:54 smithi144 ceph-mon[49149]: pgmap v362: 97 pgs: 97 active+clean; 232 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-16T08:26:54.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:26:54 smithi144 ceph-mon[49149]: from='client.34576 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T08:26:54.349 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:26:54 smithi028 ceph-mon[64115]: pgmap v362: 97 pgs: 97 active+clean; 232 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-16T08:26:54.349 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:26:54 smithi028 ceph-mon[64115]: from='client.34576 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T08:26:55.851 INFO:teuthology.orchestra.run.smithi028.stdout:{ 2024-09-16T08:26:55.851 INFO:teuthology.orchestra.run.smithi028.stdout: "mon": { 2024-09-16T08:26:55.851 INFO:teuthology.orchestra.run.smithi028.stdout: "ceph version 19.3.0-4917-g8293d73f (8293d73f8690540e843a81caec373f9cc29cf705) squid (dev)": 2 2024-09-16T08:26:55.851 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-09-16T08:26:55.851 INFO:teuthology.orchestra.run.smithi028.stdout: "mgr": { 2024-09-16T08:26:55.851 INFO:teuthology.orchestra.run.smithi028.stdout: "ceph version 19.3.0-4917-g8293d73f (8293d73f8690540e843a81caec373f9cc29cf705) squid (dev)": 2 2024-09-16T08:26:55.852 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-09-16T08:26:55.852 INFO:teuthology.orchestra.run.smithi028.stdout: "osd": { 2024-09-16T08:26:55.852 INFO:teuthology.orchestra.run.smithi028.stdout: "ceph version 19.3.0-4917-g8293d73f (8293d73f8690540e843a81caec373f9cc29cf705) squid (dev)": 8 2024-09-16T08:26:55.852 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-09-16T08:26:55.852 INFO:teuthology.orchestra.run.smithi028.stdout: "mds": { 2024-09-16T08:26:55.852 INFO:teuthology.orchestra.run.smithi028.stdout: "ceph version 19.3.0-4917-g8293d73f (8293d73f8690540e843a81caec373f9cc29cf705) squid (dev)": 2 2024-09-16T08:26:55.852 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-09-16T08:26:55.852 INFO:teuthology.orchestra.run.smithi028.stdout: "overall": { 2024-09-16T08:26:55.852 INFO:teuthology.orchestra.run.smithi028.stdout: "ceph version 19.3.0-4917-g8293d73f (8293d73f8690540e843a81caec373f9cc29cf705) squid (dev)": 14 2024-09-16T08:26:55.852 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-09-16T08:26:55.852 INFO:teuthology.orchestra.run.smithi028.stdout:} 2024-09-16T08:26:56.167 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:26:56 smithi028 ceph-mon[64115]: pgmap v363: 97 pgs: 97 active+clean; 232 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-09-16T08:26:56.167 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:26:56 smithi028 ceph-mon[64115]: from='client.? 172.21.15.28:0/2401215156' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-16T08:26:56.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:26:56 smithi144 ceph-mon[49149]: pgmap v363: 97 pgs: 97 active+clean; 232 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-09-16T08:26:56.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:26:56 smithi144 ceph-mon[49149]: from='client.? 172.21.15.28:0/2401215156' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-16T08:26:56.530 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5622b2fe-7402-11ef-bceb-c7b262605968 -e sha1=8293d73f8690540e843a81caec373f9cc29cf705 -- bash -c 'echo "wait for servicemap items w/ changing names to refresh"' 2024-09-16T08:26:58.054 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:26:58 smithi028 ceph-mon[64115]: pgmap v364: 97 pgs: 97 active+clean; 232 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-16T08:26:58.260 INFO:teuthology.orchestra.run.smithi028.stdout:wait for servicemap items w/ changing names to refresh 2024-09-16T08:26:58.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:26:58 smithi144 ceph-mon[49149]: pgmap v364: 97 pgs: 97 active+clean; 232 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-16T08:26:58.902 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5622b2fe-7402-11ef-bceb-c7b262605968 -e sha1=8293d73f8690540e843a81caec373f9cc29cf705 -- bash -c 'sleep 60' 2024-09-16T08:27:00.197 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:27:00 smithi028 ceph-mon[64115]: pgmap v365: 97 pgs: 97 active+clean; 232 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-16T08:27:00.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:27:00 smithi144 ceph-mon[49149]: pgmap v365: 97 pgs: 97 active+clean; 232 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-16T08:27:02.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:27:02 smithi144 ceph-mon[49149]: pgmap v366: 97 pgs: 97 active+clean; 232 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-09-16T08:27:02.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:27:02 smithi028 ceph-mon[64115]: pgmap v366: 97 pgs: 97 active+clean; 232 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-09-16T08:27:03.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:27:03 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-16T08:27:03.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:27:03 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-16T08:27:04.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:27:04 smithi144 ceph-mon[49149]: pgmap v367: 97 pgs: 97 active+clean; 232 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-16T08:27:04.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:27:04 smithi028 ceph-mon[64115]: pgmap v367: 97 pgs: 97 active+clean; 232 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-16T08:27:06.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:27:06 smithi144 ceph-mon[49149]: pgmap v368: 97 pgs: 97 active+clean; 232 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-09-16T08:27:06.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:27:06 smithi028 ceph-mon[64115]: pgmap v368: 97 pgs: 97 active+clean; 232 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-09-16T08:27:08.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:27:08 smithi144 ceph-mon[49149]: pgmap v369: 97 pgs: 97 active+clean; 232 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-16T08:27:08.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:27:08 smithi028 ceph-mon[64115]: pgmap v369: 97 pgs: 97 active+clean; 232 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-16T08:27:10.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:27:10 smithi144 ceph-mon[49149]: pgmap v370: 97 pgs: 97 active+clean; 232 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-16T08:27:10.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:27:10 smithi028 ceph-mon[64115]: pgmap v370: 97 pgs: 97 active+clean; 232 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-16T08:27:12.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:27:12 smithi144 ceph-mon[49149]: pgmap v371: 97 pgs: 97 active+clean; 232 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-09-16T08:27:12.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:27:12 smithi028 ceph-mon[64115]: pgmap v371: 97 pgs: 97 active+clean; 232 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-09-16T08:27:14.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:27:14 smithi144 ceph-mon[49149]: pgmap v372: 97 pgs: 97 active+clean; 232 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-16T08:27:14.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:27:14 smithi028 ceph-mon[64115]: pgmap v372: 97 pgs: 97 active+clean; 232 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-16T08:27:16.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:27:16 smithi144 ceph-mon[49149]: pgmap v373: 97 pgs: 97 active+clean; 232 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-09-16T08:27:16.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:27:16 smithi028 ceph-mon[64115]: pgmap v373: 97 pgs: 97 active+clean; 232 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-09-16T08:27:18.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:27:18 smithi144 ceph-mon[49149]: pgmap v374: 97 pgs: 97 active+clean; 232 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-16T08:27:18.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:27:18 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-16T08:27:18.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:27:18 smithi028 ceph-mon[64115]: pgmap v374: 97 pgs: 97 active+clean; 232 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-16T08:27:18.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:27:18 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-16T08:27:20.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:27:20 smithi144 ceph-mon[49149]: pgmap v375: 97 pgs: 97 active+clean; 232 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-16T08:27:20.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:27:20 smithi028 ceph-mon[64115]: pgmap v375: 97 pgs: 97 active+clean; 232 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-16T08:27:22.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:27:22 smithi028 ceph-mon[64115]: pgmap v376: 97 pgs: 97 active+clean; 232 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-09-16T08:27:22.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:27:22 smithi144 ceph-mon[49149]: pgmap v376: 97 pgs: 97 active+clean; 232 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-09-16T08:27:24.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:27:24 smithi028 ceph-mon[64115]: pgmap v377: 97 pgs: 97 active+clean; 232 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-16T08:27:24.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:27:24 smithi144 ceph-mon[49149]: pgmap v377: 97 pgs: 97 active+clean; 232 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-16T08:27:26.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:27:26 smithi028 ceph-mon[64115]: pgmap v378: 97 pgs: 97 active+clean; 232 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-09-16T08:27:26.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:27:26 smithi144 ceph-mon[49149]: pgmap v378: 97 pgs: 97 active+clean; 232 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-09-16T08:27:28.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:27:28 smithi028 ceph-mon[64115]: pgmap v379: 97 pgs: 97 active+clean; 232 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-16T08:27:28.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:27:28 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-16T08:27:28.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:27:28 smithi144 ceph-mon[49149]: pgmap v379: 97 pgs: 97 active+clean; 232 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-16T08:27:28.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:27:28 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-16T08:27:29.697 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:27:29 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:27:29.697 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:27:29 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:27:29.698 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:27:29 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:27:29.698 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:27:29 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:27:29.698 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:27:29 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-16T08:27:29.698 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:27:29 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-16T08:27:29.698 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:27:29 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:27:29.698 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:27:29 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:27:29.698 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:27:29 smithi028 ceph-mon[64115]: pgmap v380: 97 pgs: 97 active+clean; 232 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-16T08:27:29.832 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:27:29 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:27:29.832 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:27:29 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:27:29.832 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:27:29 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:27:29.832 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:27:29 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:27:29.832 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:27:29 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-16T08:27:29.832 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:27:29 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-16T08:27:29.832 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:27:29 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:27:29.832 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:27:29 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:27:29.833 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:27:29 smithi144 ceph-mon[49149]: pgmap v380: 97 pgs: 97 active+clean; 232 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-16T08:27:32.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:27:32 smithi144 ceph-mon[49149]: pgmap v381: 97 pgs: 97 active+clean; 232 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-09-16T08:27:32.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:27:32 smithi028 ceph-mon[64115]: pgmap v381: 97 pgs: 97 active+clean; 232 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-09-16T08:27:33.197 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:27:33 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-16T08:27:33.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:27:33 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-16T08:27:34.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:27:34 smithi144 ceph-mon[49149]: pgmap v382: 97 pgs: 97 active+clean; 232 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-16T08:27:34.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:27:34 smithi028 ceph-mon[64115]: pgmap v382: 97 pgs: 97 active+clean; 232 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-16T08:27:36.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:27:36 smithi144 ceph-mon[49149]: pgmap v383: 97 pgs: 97 active+clean; 232 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-09-16T08:27:36.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:27:36 smithi028 ceph-mon[64115]: pgmap v383: 97 pgs: 97 active+clean; 232 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-09-16T08:27:38.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:27:38 smithi144 ceph-mon[49149]: pgmap v384: 97 pgs: 97 active+clean; 232 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-16T08:27:38.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:27:38 smithi028 ceph-mon[64115]: pgmap v384: 97 pgs: 97 active+clean; 232 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-16T08:27:40.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:27:40 smithi144 ceph-mon[49149]: pgmap v385: 97 pgs: 97 active+clean; 232 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-16T08:27:40.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:27:40 smithi028 ceph-mon[64115]: pgmap v385: 97 pgs: 97 active+clean; 232 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-16T08:27:42.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:27:42 smithi144 ceph-mon[49149]: pgmap v386: 97 pgs: 97 active+clean; 232 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-09-16T08:27:42.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:27:42 smithi028 ceph-mon[64115]: pgmap v386: 97 pgs: 97 active+clean; 232 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-09-16T08:27:44.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:27:44 smithi144 ceph-mon[49149]: pgmap v387: 97 pgs: 97 active+clean; 232 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-16T08:27:44.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:27:44 smithi028 ceph-mon[64115]: pgmap v387: 97 pgs: 97 active+clean; 232 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-16T08:27:46.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:27:46 smithi144 ceph-mon[49149]: pgmap v388: 97 pgs: 97 active+clean; 232 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-09-16T08:27:46.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:27:46 smithi028 ceph-mon[64115]: pgmap v388: 97 pgs: 97 active+clean; 232 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-09-16T08:27:48.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:27:48 smithi144 ceph-mon[49149]: pgmap v389: 97 pgs: 97 active+clean; 232 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-16T08:27:48.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:27:48 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-16T08:27:48.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:27:48 smithi028 ceph-mon[64115]: pgmap v389: 97 pgs: 97 active+clean; 232 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-16T08:27:48.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:27:48 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-16T08:27:50.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:27:50 smithi144 ceph-mon[49149]: pgmap v390: 97 pgs: 97 active+clean; 232 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-16T08:27:50.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:27:50 smithi028 ceph-mon[64115]: pgmap v390: 97 pgs: 97 active+clean; 232 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-16T08:27:52.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:27:52 smithi144 ceph-mon[49149]: pgmap v391: 97 pgs: 97 active+clean; 232 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-09-16T08:27:52.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:27:52 smithi028 ceph-mon[64115]: pgmap v391: 97 pgs: 97 active+clean; 232 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-09-16T08:27:54.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:27:54 smithi144 ceph-mon[49149]: pgmap v392: 97 pgs: 97 active+clean; 232 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-16T08:27:54.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:27:54 smithi028 ceph-mon[64115]: pgmap v392: 97 pgs: 97 active+clean; 232 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-16T08:27:56.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:27:56 smithi144 ceph-mon[49149]: pgmap v393: 97 pgs: 97 active+clean; 232 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-09-16T08:27:56.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:27:56 smithi028 ceph-mon[64115]: pgmap v393: 97 pgs: 97 active+clean; 232 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-09-16T08:27:58.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:27:58 smithi144 ceph-mon[49149]: pgmap v394: 97 pgs: 97 active+clean; 232 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-16T08:27:58.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:27:58 smithi028 ceph-mon[64115]: pgmap v394: 97 pgs: 97 active+clean; 232 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-16T08:28:00.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:28:00 smithi144 ceph-mon[49149]: pgmap v395: 97 pgs: 97 active+clean; 232 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-16T08:28:00.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:28:00 smithi028 ceph-mon[64115]: pgmap v395: 97 pgs: 97 active+clean; 232 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-16T08:28:01.290 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5622b2fe-7402-11ef-bceb-c7b262605968 -e sha1=8293d73f8690540e843a81caec373f9cc29cf705 -- bash -c 'ceph orch ps' 2024-09-16T08:28:02.197 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:28:02 smithi028 ceph-mon[64115]: pgmap v396: 97 pgs: 97 active+clean; 232 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-09-16T08:28:02.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:28:02 smithi144 ceph-mon[49149]: pgmap v396: 97 pgs: 97 active+clean; 232 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-09-16T08:28:03.197 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:28:03 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-16T08:28:03.324 INFO:teuthology.orchestra.run.smithi028.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-16T08:28:03.324 INFO:teuthology.orchestra.run.smithi028.stdout:alertmanager.smithi028 smithi028 *:9093,9094 running (12m) 102s ago 21m 22.2M - 0.27.0 11f11916f8cd a14fcb4cb2b6 2024-09-16T08:28:03.324 INFO:teuthology.orchestra.run.smithi028.stdout:crash.smithi028 smithi028 running (9m) 102s ago 21m 6899k - 19.3.0-4917-g8293d73f 425e316ba8eb 4bb89df2a867 2024-09-16T08:28:03.324 INFO:teuthology.orchestra.run.smithi028.stdout:crash.smithi144 smithi144 running (9m) 2m ago 20m 6895k - 19.3.0-4917-g8293d73f 425e316ba8eb 24a96875d7af 2024-09-16T08:28:03.324 INFO:teuthology.orchestra.run.smithi028.stdout:grafana.smithi028 smithi028 *:3000 running (108s) 102s ago 21m 38.7M - 10.4.0 c8b91775d855 79711ecc8811 2024-09-16T08:28:03.324 INFO:teuthology.orchestra.run.smithi028.stdout:mds.foofs.smithi028.srtgev smithi028 running (2m) 102s ago 15m 32.7M - 19.3.0-4917-g8293d73f 425e316ba8eb 102d36a11737 2024-09-16T08:28:03.325 INFO:teuthology.orchestra.run.smithi028.stdout:mds.foofs.smithi144.oozgsi smithi144 running (2m) 2m ago 15m 13.0M - 19.3.0-4917-g8293d73f 425e316ba8eb 9917dfd27777 2024-09-16T08:28:03.325 INFO:teuthology.orchestra.run.smithi028.stdout:mgr.smithi028.azvyzy smithi028 *:8443,9283,8765 running (11m) 102s ago 22m 533M - 19.3.0-4917-g8293d73f 425e316ba8eb e01d4382d03f 2024-09-16T08:28:03.325 INFO:teuthology.orchestra.run.smithi028.stdout:mgr.smithi144.utzszj smithi144 *:8443,9283,8765 running (10m) 2m ago 20m 458M - 19.3.0-4917-g8293d73f 425e316ba8eb 71f7f802bffb 2024-09-16T08:28:03.325 INFO:teuthology.orchestra.run.smithi028.stdout:mon.smithi028 smithi028 running (10m) 102s ago 22m 63.6M 2048M 19.3.0-4917-g8293d73f 425e316ba8eb 14efcedc0088 2024-09-16T08:28:03.325 INFO:teuthology.orchestra.run.smithi028.stdout:mon.smithi144 smithi144 running (9m) 2m ago 20m 51.3M 2048M 19.3.0-4917-g8293d73f 425e316ba8eb c081c5ed44b5 2024-09-16T08:28:03.325 INFO:teuthology.orchestra.run.smithi028.stdout:nfs.foo.0.0.smithi028.dkttgj smithi028 *:2049 running (2m) 102s ago 15m 63.0M - 5.9 425e316ba8eb 8a211930c928 2024-09-16T08:28:03.325 INFO:teuthology.orchestra.run.smithi028.stdout:nfs.foo.1.0.smithi144.epipkg smithi144 *:2049 running (2m) 2m ago 15m 19.8M - 5.9 425e316ba8eb 1a56ade0da3e 2024-09-16T08:28:03.325 INFO:teuthology.orchestra.run.smithi028.stdout:node-exporter.smithi028 smithi028 *:9100 running (12m) 102s ago 21m 11.7M - 1.7.0 72c9c2088986 9e2b8e8a63a9 2024-09-16T08:28:03.325 INFO:teuthology.orchestra.run.smithi028.stdout:node-exporter.smithi144 smithi144 *:9100 running (11m) 2m ago 20m 10.2M - 1.7.0 72c9c2088986 78834c5e992f 2024-09-16T08:28:03.325 INFO:teuthology.orchestra.run.smithi028.stdout:osd.0 smithi028 running (8m) 102s ago 19m 429M 1937M 19.3.0-4917-g8293d73f 425e316ba8eb 2fc02ba4f14f 2024-09-16T08:28:03.325 INFO:teuthology.orchestra.run.smithi028.stdout:osd.1 smithi028 running (7m) 102s ago 19m 417M 1937M 19.3.0-4917-g8293d73f 425e316ba8eb df34bd603d8f 2024-09-16T08:28:03.325 INFO:teuthology.orchestra.run.smithi028.stdout:osd.2 smithi028 running (6m) 102s ago 19m 309M 1937M 19.3.0-4917-g8293d73f 425e316ba8eb c92be6327b12 2024-09-16T08:28:03.325 INFO:teuthology.orchestra.run.smithi028.stdout:osd.3 smithi028 running (6m) 102s ago 18m 417M 1937M 19.3.0-4917-g8293d73f 425e316ba8eb bd84ba1224f2 2024-09-16T08:28:03.325 INFO:teuthology.orchestra.run.smithi028.stdout:osd.4 smithi144 running (5m) 2m ago 18m 340M 2704M 19.3.0-4917-g8293d73f 425e316ba8eb c48cb68e901e 2024-09-16T08:28:03.326 INFO:teuthology.orchestra.run.smithi028.stdout:osd.5 smithi144 running (4m) 2m ago 17m 335M 2704M 19.3.0-4917-g8293d73f 425e316ba8eb 59ec97fb3cf3 2024-09-16T08:28:03.326 INFO:teuthology.orchestra.run.smithi028.stdout:osd.6 smithi144 running (3m) 2m ago 17m 307M 2704M 19.3.0-4917-g8293d73f 425e316ba8eb b301bf0731f0 2024-09-16T08:28:03.326 INFO:teuthology.orchestra.run.smithi028.stdout:osd.7 smithi144 running (3m) 2m ago 17m 292M 2704M 19.3.0-4917-g8293d73f 425e316ba8eb f1a5cbf3a7bd 2024-09-16T08:28:03.326 INFO:teuthology.orchestra.run.smithi028.stdout:prometheus.smithi028 smithi028 *:9095 running (10m) 102s ago 20m 56.3M - 2.51.0 1d3b7f56885b 42667809df2b 2024-09-16T08:28:03.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:28:03 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-16T08:28:03.942 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5622b2fe-7402-11ef-bceb-c7b262605968 -e sha1=8293d73f8690540e843a81caec373f9cc29cf705 -- bash -c 'ceph orch upgrade status' 2024-09-16T08:28:04.197 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:28:04 smithi028 ceph-mon[64115]: pgmap v397: 97 pgs: 97 active+clean; 232 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-16T08:28:04.331 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:28:04 smithi144 ceph-mon[49149]: pgmap v397: 97 pgs: 97 active+clean; 232 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-16T08:28:05.180 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:28:05 smithi028 ceph-mon[64115]: from='client.34584 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T08:28:05.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:28:05 smithi144 ceph-mon[49149]: from='client.34584 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T08:28:05.954 INFO:teuthology.orchestra.run.smithi028.stdout:{ 2024-09-16T08:28:05.954 INFO:teuthology.orchestra.run.smithi028.stdout: "target_image": null, 2024-09-16T08:28:05.954 INFO:teuthology.orchestra.run.smithi028.stdout: "in_progress": false, 2024-09-16T08:28:05.954 INFO:teuthology.orchestra.run.smithi028.stdout: "which": "", 2024-09-16T08:28:05.954 INFO:teuthology.orchestra.run.smithi028.stdout: "services_complete": [], 2024-09-16T08:28:05.954 INFO:teuthology.orchestra.run.smithi028.stdout: "progress": null, 2024-09-16T08:28:05.955 INFO:teuthology.orchestra.run.smithi028.stdout: "message": "", 2024-09-16T08:28:05.955 INFO:teuthology.orchestra.run.smithi028.stdout: "is_paused": false 2024-09-16T08:28:05.955 INFO:teuthology.orchestra.run.smithi028.stdout:} 2024-09-16T08:28:06.332 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:28:06 smithi144 ceph-mon[49149]: pgmap v398: 97 pgs: 97 active+clean; 232 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-09-16T08:28:06.376 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:28:06 smithi028 ceph-mon[64115]: pgmap v398: 97 pgs: 97 active+clean; 232 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-09-16T08:28:06.562 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5622b2fe-7402-11ef-bceb-c7b262605968 -e sha1=8293d73f8690540e843a81caec373f9cc29cf705 -- bash -c 'ceph health detail' 2024-09-16T08:28:07.182 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:28:07 smithi028 ceph-mon[64115]: from='client.34588 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T08:28:07.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:28:07 smithi144 ceph-mon[49149]: from='client.34588 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T08:28:08.185 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:28:08 smithi028 ceph-mon[64115]: pgmap v399: 97 pgs: 97 active+clean; 232 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-16T08:28:08.585 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:28:08 smithi144 ceph-mon[49149]: pgmap v399: 97 pgs: 97 active+clean; 232 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-16T08:28:08.705 INFO:teuthology.orchestra.run.smithi028.stdout:HEALTH_OK 2024-09-16T08:28:09.318 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:28:09 smithi028 ceph-mon[64115]: from='client.? 172.21.15.28:0/3872927514' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-16T08:28:09.345 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5622b2fe-7402-11ef-bceb-c7b262605968 -e sha1=8293d73f8690540e843a81caec373f9cc29cf705 -- bash -c 'ceph versions' 2024-09-16T08:28:09.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:28:09 smithi144 ceph-mon[49149]: from='client.? 172.21.15.28:0/3872927514' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-16T08:28:10.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:28:10 smithi028 ceph-mon[64115]: pgmap v400: 97 pgs: 97 active+clean; 232 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-16T08:28:10.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:28:10 smithi144 ceph-mon[49149]: pgmap v400: 97 pgs: 97 active+clean; 232 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-16T08:28:11.441 INFO:teuthology.orchestra.run.smithi028.stdout:{ 2024-09-16T08:28:11.441 INFO:teuthology.orchestra.run.smithi028.stdout: "mon": { 2024-09-16T08:28:11.442 INFO:teuthology.orchestra.run.smithi028.stdout: "ceph version 19.3.0-4917-g8293d73f (8293d73f8690540e843a81caec373f9cc29cf705) squid (dev)": 2 2024-09-16T08:28:11.442 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-09-16T08:28:11.442 INFO:teuthology.orchestra.run.smithi028.stdout: "mgr": { 2024-09-16T08:28:11.442 INFO:teuthology.orchestra.run.smithi028.stdout: "ceph version 19.3.0-4917-g8293d73f (8293d73f8690540e843a81caec373f9cc29cf705) squid (dev)": 2 2024-09-16T08:28:11.442 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-09-16T08:28:11.442 INFO:teuthology.orchestra.run.smithi028.stdout: "osd": { 2024-09-16T08:28:11.442 INFO:teuthology.orchestra.run.smithi028.stdout: "ceph version 19.3.0-4917-g8293d73f (8293d73f8690540e843a81caec373f9cc29cf705) squid (dev)": 8 2024-09-16T08:28:11.442 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-09-16T08:28:11.442 INFO:teuthology.orchestra.run.smithi028.stdout: "mds": { 2024-09-16T08:28:11.442 INFO:teuthology.orchestra.run.smithi028.stdout: "ceph version 19.3.0-4917-g8293d73f (8293d73f8690540e843a81caec373f9cc29cf705) squid (dev)": 2 2024-09-16T08:28:11.442 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-09-16T08:28:11.442 INFO:teuthology.orchestra.run.smithi028.stdout: "overall": { 2024-09-16T08:28:11.443 INFO:teuthology.orchestra.run.smithi028.stdout: "ceph version 19.3.0-4917-g8293d73f (8293d73f8690540e843a81caec373f9cc29cf705) squid (dev)": 14 2024-09-16T08:28:11.443 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-09-16T08:28:11.443 INFO:teuthology.orchestra.run.smithi028.stdout:} 2024-09-16T08:28:12.042 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5622b2fe-7402-11ef-bceb-c7b262605968 -e sha1=8293d73f8690540e843a81caec373f9cc29cf705 -- bash -c 'ceph versions | jq -e '"'"'.overall | length == 1'"'"'' 2024-09-16T08:28:12.349 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:28:12 smithi028 ceph-mon[64115]: pgmap v401: 97 pgs: 97 active+clean; 232 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-09-16T08:28:12.349 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:28:12 smithi028 ceph-mon[64115]: from='client.? 172.21.15.28:0/52796543' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-16T08:28:12.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:28:12 smithi144 ceph-mon[49149]: pgmap v401: 97 pgs: 97 active+clean; 232 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-09-16T08:28:12.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:28:12 smithi144 ceph-mon[49149]: from='client.? 172.21.15.28:0/52796543' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-16T08:28:14.156 INFO:teuthology.orchestra.run.smithi028.stdout:true 2024-09-16T08:28:14.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:28:14 smithi028 ceph-mon[64115]: pgmap v402: 97 pgs: 97 active+clean; 232 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-16T08:28:14.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:28:14 smithi144 ceph-mon[49149]: pgmap v402: 97 pgs: 97 active+clean; 232 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-16T08:28:14.891 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5622b2fe-7402-11ef-bceb-c7b262605968 -e sha1=8293d73f8690540e843a81caec373f9cc29cf705 -- bash -c 'ceph versions | jq -e '"'"'.overall | keys'"'"' | grep $sha1' 2024-09-16T08:28:15.196 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:28:15 smithi028 ceph-mon[64115]: from='client.? 172.21.15.28:0/1777746229' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-16T08:28:15.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:28:15 smithi144 ceph-mon[49149]: from='client.? 172.21.15.28:0/1777746229' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-16T08:28:16.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:28:16 smithi028 ceph-mon[64115]: pgmap v403: 97 pgs: 97 active+clean; 232 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-09-16T08:28:16.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:28:16 smithi144 ceph-mon[49149]: pgmap v403: 97 pgs: 97 active+clean; 232 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-09-16T08:28:17.023 INFO:teuthology.orchestra.run.smithi028.stdout: "ceph version 19.3.0-4917-g8293d73f (8293d73f8690540e843a81caec373f9cc29cf705) squid (dev)" 2024-09-16T08:28:17.385 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:28:17 smithi028 ceph-mon[64115]: from='client.? 172.21.15.28:0/1882813482' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-16T08:28:17.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:28:17 smithi144 ceph-mon[49149]: from='client.? 172.21.15.28:0/1882813482' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-16T08:28:17.670 INFO:teuthology.task.sequential:In sequential, running task cephadm.wait_for_service... 2024-09-16T08:28:17.671 INFO:tasks.cephadm:Waiting for ceph service nfs.foo to start (timeout 300)... 2024-09-16T08:28:17.672 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5622b2fe-7402-11ef-bceb-c7b262605968 -- ceph orch ls -f json 2024-09-16T08:28:18.295 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:28:18 smithi028 ceph-mon[64115]: pgmap v404: 97 pgs: 97 active+clean; 232 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-16T08:28:18.295 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:28:18 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-16T08:28:18.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:28:18 smithi144 ceph-mon[49149]: pgmap v404: 97 pgs: 97 active+clean; 232 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-16T08:28:18.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:28:18 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-16T08:28:19.704 INFO:teuthology.orchestra.run.smithi028.stdout: 2024-09-16T08:28:19.704 INFO:teuthology.orchestra.run.smithi028.stdout:[{"placement": {"count": 1}, "service_name": "alertmanager", "service_type": "alertmanager", "status": {"created": "2024-09-16T08:06:03.680757Z", "last_refresh": "2024-09-16T08:26:20.389817Z", "ports": [9093, 9094], "running": 1, "size": 1}}, {"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"created": "2024-09-16T08:05:58.113512Z", "last_refresh": "2024-09-16T08:25:54.097794Z", "running": 2, "size": 2}}, {"placement": {"count": 1}, "service_name": "grafana", "service_type": "grafana", "spec": {"anonymous_access": true, "protocol": "https"}, "status": {"created": "2024-09-16T08:06:00.803468Z", "last_refresh": "2024-09-16T08:26:20.389954Z", "ports": [3000], "running": 1, "size": 1}}, {"placement": {"count": 2}, "service_id": "foofs", "service_name": "mds.foofs", "service_type": "mds", "status": {"created": "2024-09-16T08:12:22.255272Z", "last_refresh": "2024-09-16T08:25:54.098366Z", "running": 2, "size": 2}}, {"placement": {"count": 2}, "service_name": "mgr", "service_type": "mgr", "status": {"created": "2024-09-16T08:05:56.955187Z", "last_refresh": "2024-09-16T08:25:54.097889Z", "running": 2, "size": 2}}, {"placement": {"count": 2, "hosts": ["smithi028:172.21.15.28=smithi028", "smithi144:172.21.15.144=smithi144"]}, "service_name": "mon", "service_type": "mon", "status": {"created": "2024-09-16T08:06:46.314972Z", "last_refresh": "2024-09-16T08:25:54.097963Z", "running": 2, "size": 2}}, {"events": ["2024-09-16T08:27:28.695560Z service:nfs.foo [INFO] \"service was created\""], "placement": {"count": 2}, "service_id": "foo", "service_name": "nfs.foo", "service_type": "nfs", "spec": {"port": 2049}, "status": {"created": "2024-09-16T08:12:45.278870Z", "last_refresh": "2024-09-16T08:25:54.098437Z", "ports": [2049], "running": 2, "size": 2}}, {"placement": {"host_pattern": "*"}, "service_name": "node-exporter", "service_type": "node-exporter", "status": {"created": "2024-09-16T08:06:02.516246Z", "last_refresh": "2024-09-16T08:25:54.098032Z", "ports": [9100], "running": 2, "size": 2}}, {"service_name": "osd", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "425e316ba8ebf74f04aa3640f7deb4fa3f293a78d0c8d097cb679eaf86ddca6c", "container_image_name": "quay.ceph.io/ceph-ci/ceph@sha256:f7ad55f9ccd17ba66aadb19b4d291e67b60ea24e5d8a0c0e92941fd59c81e4ce", "last_refresh": "2024-09-16T08:25:54.098101Z", "running": 8, "size": 8}, "unmanaged": true}, {"placement": {"count": 1}, "service_name": "prometheus", "service_type": "prometheus", "status": {"created": "2024-09-16T08:05:59.457450Z", "last_refresh": "2024-09-16T08:26:20.390092Z", "ports": [9095], "running": 1, "size": 1}}] 2024-09-16T08:28:20.340 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:28:20 smithi028 ceph-mon[64115]: pgmap v405: 97 pgs: 97 active+clean; 232 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-16T08:28:20.369 INFO:tasks.cephadm:nfs.foo has 2/2 2024-09-16T08:28:20.370 DEBUG:teuthology.parallel:result is None 2024-09-16T08:28:20.370 INFO:teuthology.run_tasks:Running task vip.exec... 2024-09-16T08:28:20.383 INFO:tasks.vip:Running commands on role host.a host ubuntu@smithi028.front.sepia.ceph.com 2024-09-16T08:28:20.383 DEBUG:teuthology.orchestra.run.smithi028:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c 'umount /mnt/foo' 2024-09-16T08:28:20.414 INFO:teuthology.orchestra.run.smithi028.stderr:+ umount /mnt/foo 2024-09-16T08:28:20.436 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-09-16T08:28:20.446 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi028.front.sepia.ceph.com 2024-09-16T08:28:20.447 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5622b2fe-7402-11ef-bceb-c7b262605968 -- bash -c 'ceph nfs cluster ls | grep foo' 2024-09-16T08:28:20.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:28:20 smithi144 ceph-mon[49149]: pgmap v405: 97 pgs: 97 active+clean; 232 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-16T08:28:21.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:28:21 smithi028 ceph-mon[64115]: from='client.34608 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-16T08:28:21.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:28:21 smithi144 ceph-mon[49149]: from='client.34608 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-16T08:28:22.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:28:22 smithi028 ceph-mon[64115]: pgmap v406: 97 pgs: 97 active+clean; 232 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-09-16T08:28:22.482 INFO:teuthology.orchestra.run.smithi028.stdout: "foo" 2024-09-16T08:28:22.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:28:22 smithi144 ceph-mon[49149]: pgmap v406: 97 pgs: 97 active+clean; 232 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-09-16T08:28:23.088 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5622b2fe-7402-11ef-bceb-c7b262605968 -- bash -c 'ceph nfs export ls foo --detailed' 2024-09-16T08:28:24.302 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:28:24 smithi028 ceph-mon[64115]: from='client.34612 -' entity='client.admin' cmd=[{"prefix": "nfs cluster ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T08:28:24.302 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:28:24 smithi028 ceph-mon[64115]: pgmap v407: 97 pgs: 97 active+clean; 232 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-16T08:28:24.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:28:24 smithi144 ceph-mon[49149]: from='client.34612 -' entity='client.admin' cmd=[{"prefix": "nfs cluster ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T08:28:24.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:28:24 smithi144 ceph-mon[49149]: pgmap v407: 97 pgs: 97 active+clean; 232 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-16T08:28:25.108 INFO:teuthology.orchestra.run.smithi028.stdout:[ 2024-09-16T08:28:25.109 INFO:teuthology.orchestra.run.smithi028.stdout: { 2024-09-16T08:28:25.109 INFO:teuthology.orchestra.run.smithi028.stdout: "access_type": "RW", 2024-09-16T08:28:25.109 INFO:teuthology.orchestra.run.smithi028.stdout: "clients": [], 2024-09-16T08:28:25.109 INFO:teuthology.orchestra.run.smithi028.stdout: "cluster_id": "foo", 2024-09-16T08:28:25.109 INFO:teuthology.orchestra.run.smithi028.stdout: "export_id": 1, 2024-09-16T08:28:25.109 INFO:teuthology.orchestra.run.smithi028.stdout: "fsal": { 2024-09-16T08:28:25.109 INFO:teuthology.orchestra.run.smithi028.stdout: "fs_name": "foofs", 2024-09-16T08:28:25.109 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "CEPH", 2024-09-16T08:28:25.109 INFO:teuthology.orchestra.run.smithi028.stdout: "user_id": "nfs.foo.1" 2024-09-16T08:28:25.109 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-09-16T08:28:25.109 INFO:teuthology.orchestra.run.smithi028.stdout: "path": "/", 2024-09-16T08:28:25.109 INFO:teuthology.orchestra.run.smithi028.stdout: "protocols": [ 2024-09-16T08:28:25.110 INFO:teuthology.orchestra.run.smithi028.stdout: 4 2024-09-16T08:28:25.110 INFO:teuthology.orchestra.run.smithi028.stdout: ], 2024-09-16T08:28:25.110 INFO:teuthology.orchestra.run.smithi028.stdout: "pseudo": "/fake", 2024-09-16T08:28:25.110 INFO:teuthology.orchestra.run.smithi028.stdout: "security_label": true, 2024-09-16T08:28:25.110 INFO:teuthology.orchestra.run.smithi028.stdout: "squash": "none", 2024-09-16T08:28:25.110 INFO:teuthology.orchestra.run.smithi028.stdout: "transports": [ 2024-09-16T08:28:25.110 INFO:teuthology.orchestra.run.smithi028.stdout: "TCP" 2024-09-16T08:28:25.110 INFO:teuthology.orchestra.run.smithi028.stdout: ] 2024-09-16T08:28:25.110 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-09-16T08:28:25.110 INFO:teuthology.orchestra.run.smithi028.stdout:] 2024-09-16T08:28:25.675 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5622b2fe-7402-11ef-bceb-c7b262605968 -- bash -c 'rados -p .nfs --all ls -' 2024-09-16T08:28:26.231 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:28:26 smithi028 ceph-mon[64115]: pgmap v408: 97 pgs: 97 active+clean; 232 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-09-16T08:28:26.231 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:28:26 smithi028 ceph-mon[64115]: from='client.34616 -' entity='client.admin' cmd=[{"prefix": "nfs export ls", "cluster_id": "foo", "detailed": true, "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T08:28:26.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:28:26 smithi144 ceph-mon[49149]: pgmap v408: 97 pgs: 97 active+clean; 232 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-09-16T08:28:26.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:28:26 smithi144 ceph-mon[49149]: from='client.34616 -' entity='client.admin' cmd=[{"prefix": "nfs export ls", "cluster_id": "foo", "detailed": true, "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T08:28:27.344 INFO:teuthology.orchestra.run.smithi028.stdout:foo rec-0000000000000004:nfs.foo.1 2024-09-16T08:28:27.344 INFO:teuthology.orchestra.run.smithi028.stdout:foo grace 2024-09-16T08:28:27.344 INFO:teuthology.orchestra.run.smithi028.stdout:foo rec-0000000000000004:nfs.foo.0 2024-09-16T08:28:27.344 INFO:teuthology.orchestra.run.smithi028.stdout:foo conf-nfs.foo 2024-09-16T08:28:27.344 INFO:teuthology.orchestra.run.smithi028.stdout:foo export-1 2024-09-16T08:28:27.840 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-09-16T08:28:27.851 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi028.front.sepia.ceph.com 2024-09-16T08:28:27.851 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5622b2fe-7402-11ef-bceb-c7b262605968 -- bash -c 'set -ex 2024-09-16T08:28:27.851 DEBUG:teuthology.orchestra.run.smithi028:> [[ `ceph config get mgr mgr/cephadm/migration_current` -gt 2 ]] 2024-09-16T08:28:27.851 DEBUG:teuthology.orchestra.run.smithi028:> ' 2024-09-16T08:28:28.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:28:28 smithi028 ceph-mon[64115]: pgmap v409: 97 pgs: 97 active+clean; 232 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-09-16T08:28:28.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:28:28 smithi028 ceph-mon[64115]: mgrmap e44: smithi028.azvyzy(active, since 10m), standbys: smithi144.utzszj 2024-09-16T08:28:28.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:28:28 smithi144 ceph-mon[49149]: pgmap v409: 97 pgs: 97 active+clean; 232 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-09-16T08:28:28.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:28:28 smithi144 ceph-mon[49149]: mgrmap e44: smithi028.azvyzy(active, since 10m), standbys: smithi144.utzszj 2024-09-16T08:28:29.197 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:28:29 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-16T08:28:29.457 INFO:teuthology.orchestra.run.smithi028.stderr:++ ceph config get mgr mgr/cephadm/migration_current 2024-09-16T08:28:29.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:28:29 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-16T08:28:29.745 INFO:teuthology.orchestra.run.smithi028.stderr:+ [[ 8 -gt 2 ]] 2024-09-16T08:28:30.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:28:30 smithi028 ceph-mon[64115]: pgmap v410: 97 pgs: 97 active+clean; 232 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-09-16T08:28:30.447 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:28:30 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-16T08:28:30.448 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:28:30 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-16T08:28:30.448 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:28:30 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:28:30.448 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:28:30 smithi028 ceph-mon[64115]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:28:30.448 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:28:30 smithi028 ceph-mon[64115]: from='client.? 172.21.15.28:0/2692777466' entity='client.admin' cmd={"prefix": "config get", "who": "mgr", "key": "mgr/cephadm/migration_current"} : dispatch 2024-09-16T08:28:30.477 DEBUG:teuthology.run_tasks:Unwinding manager cephadm 2024-09-16T08:28:30.486 INFO:tasks.cephadm:Teardown begin 2024-09-16T08:28:30.487 DEBUG:teuthology.orchestra.run.smithi028:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-09-16T08:28:30.515 DEBUG:teuthology.orchestra.run.smithi144:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-09-16T08:28:30.539 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:28:30 smithi144 ceph-mon[49149]: pgmap v410: 97 pgs: 97 active+clean; 232 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-09-16T08:28:30.539 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:28:30 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-16T08:28:30.539 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:28:30 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-16T08:28:30.539 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:28:30 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:28:30.539 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:28:30 smithi144 ceph-mon[49149]: from='mgr.34104 172.21.15.28:0/3067991022' entity='mgr.smithi028.azvyzy' 2024-09-16T08:28:30.539 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:28:30 smithi144 ceph-mon[49149]: from='client.? 172.21.15.28:0/2692777466' entity='client.admin' cmd={"prefix": "config get", "who": "mgr", "key": "mgr/cephadm/migration_current"} : dispatch 2024-09-16T08:28:30.544 INFO:tasks.cephadm:Cleaning up testdir ceph.* files... 2024-09-16T08:28:30.544 DEBUG:teuthology.orchestra.run.smithi028:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2024-09-16T08:28:30.571 DEBUG:teuthology.orchestra.run.smithi144:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2024-09-16T08:28:30.600 INFO:tasks.cephadm:Stopping all daemons... 2024-09-16T08:28:30.600 INFO:tasks.cephadm.mon.smithi028:Stopping mon.smithi028... 2024-09-16T08:28:30.600 DEBUG:teuthology.orchestra.run.smithi028:> sudo systemctl stop ceph-5622b2fe-7402-11ef-bceb-c7b262605968@mon.smithi028 2024-09-16T08:28:30.899 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:28:30 smithi028 systemd[1]: Stopping Ceph mon.smithi028 for 5622b2fe-7402-11ef-bceb-c7b262605968... 2024-09-16T08:28:31.154 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:28:30 smithi028 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-mon-smithi028[64111]: 2024-09-16T08:28:30.897+0000 7f33322ad640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.smithi028 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-09-16T08:28:31.154 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:28:30 smithi028 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-mon-smithi028[64111]: 2024-09-16T08:28:30.897+0000 7f33322ad640 -1 mon.smithi028@0(leader) e3 *** Got Signal Terminated *** 2024-09-16T08:28:31.427 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:28:31 smithi028 podman[94110]: 2024-09-16 08:28:31.154298122 +0000 UTC m=+0.348599740 container died 14efcedc0088717f01c964e67e71441d1547d4705d5236fd6e00c0beff69d513 (image=quay.ceph.io/ceph-ci/ceph@sha256:f7ad55f9ccd17ba66aadb19b4d291e67b60ea24e5d8a0c0e92941fd59c81e4ce, name=ceph-5622b2fe-7402-11ef-bceb-c7b262605968-mon-smithi028, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=wip-guits-main-2024-09-13-1248-8293d73, org.label-schema.schema-version=1.0) 2024-09-16T08:28:31.697 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:28:31 smithi028 podman[94110]: 2024-09-16 08:28:31.517526459 +0000 UTC m=+0.711828075 container cleanup 14efcedc0088717f01c964e67e71441d1547d4705d5236fd6e00c0beff69d513 (image=quay.ceph.io/ceph-ci/ceph@sha256:f7ad55f9ccd17ba66aadb19b4d291e67b60ea24e5d8a0c0e92941fd59c81e4ce, name=ceph-5622b2fe-7402-11ef-bceb-c7b262605968-mon-smithi028, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, RELEASE=wip-guits-main-2024-09-13-1248-8293d73, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.build-date=20240731) 2024-09-16T08:28:31.697 INFO:journalctl@ceph.mon.smithi028.smithi028.stdout:Sep 16 08:28:31 smithi028 bash[94110]: ceph-5622b2fe-7402-11ef-bceb-c7b262605968-mon-smithi028 2024-09-16T08:28:31.952 DEBUG:teuthology.orchestra.run.smithi028:> sudo pkill -f 'journalctl -f -n 0 -u ceph-5622b2fe-7402-11ef-bceb-c7b262605968@mon.smithi028.service' 2024-09-16T08:28:32.004 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-16T08:28:32.004 INFO:tasks.cephadm.mon.smithi028:Stopped mon.smithi028 2024-09-16T08:28:32.004 INFO:tasks.cephadm.mon.smithi144:Stopping mon.smithi144... 2024-09-16T08:28:32.005 DEBUG:teuthology.orchestra.run.smithi144:> sudo systemctl stop ceph-5622b2fe-7402-11ef-bceb-c7b262605968@mon.smithi144 2024-09-16T08:28:32.318 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:28:32 smithi144 systemd[1]: Stopping Ceph mon.smithi144 for 5622b2fe-7402-11ef-bceb-c7b262605968... 2024-09-16T08:28:32.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:28:32 smithi144 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-mon-smithi144[49145]: 2024-09-16T08:28:32.317+0000 7f4803f82640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.smithi144 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-09-16T08:28:32.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:28:32 smithi144 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-mon-smithi144[49145]: 2024-09-16T08:28:32.317+0000 7f4803f82640 -1 mon.smithi144@1(peon) e3 *** Got Signal Terminated *** 2024-09-16T08:28:32.583 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:28:32 smithi144 podman[68539]: 2024-09-16 08:28:32.481084118 +0000 UTC m=+0.274489779 container died c081c5ed44b540e028495bb1c8b3001db8efa379cb6417b527cbf19662a33ca7 (image=quay.ceph.io/ceph-ci/ceph@sha256:f7ad55f9ccd17ba66aadb19b4d291e67b60ea24e5d8a0c0e92941fd59c81e4ce, name=ceph-5622b2fe-7402-11ef-bceb-c7b262605968-mon-smithi144, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_CLEAN=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, ceph=True, RELEASE=wip-guits-main-2024-09-13-1248-8293d73, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0) 2024-09-16T08:28:33.082 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:28:32 smithi144 podman[68539]: 2024-09-16 08:28:32.909782544 +0000 UTC m=+0.703188204 container cleanup c081c5ed44b540e028495bb1c8b3001db8efa379cb6417b527cbf19662a33ca7 (image=quay.ceph.io/ceph-ci/ceph@sha256:f7ad55f9ccd17ba66aadb19b4d291e67b60ea24e5d8a0c0e92941fd59c81e4ce, name=ceph-5622b2fe-7402-11ef-bceb-c7b262605968-mon-smithi144, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , RELEASE=wip-guits-main-2024-09-13-1248-8293d73, org.label-schema.license=GPLv2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, ceph=True, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-16T08:28:33.082 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:28:32 smithi144 bash[68539]: ceph-5622b2fe-7402-11ef-bceb-c7b262605968-mon-smithi144 2024-09-16T08:28:33.412 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Sep 16 08:28:33 smithi144 podman[68554]: 2024-09-16 08:28:33.15345278 +0000 UTC m=+0.668051854 container remove c081c5ed44b540e028495bb1c8b3001db8efa379cb6417b527cbf19662a33ca7 (image=quay.ceph.io/ceph-ci/ceph@sha256:f7ad55f9ccd17ba66aadb19b4d291e67b60ea24e5d8a0c0e92941fd59c81e4ce, name=ceph-5622b2fe-7402-11ef-bceb-c7b262605968-mon-smithi144, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, ceph=True, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-guits-main-2024-09-13-1248-8293d73, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-16T08:28:33.426 DEBUG:teuthology.orchestra.run.smithi144:> sudo pkill -f 'journalctl -f -n 0 -u ceph-5622b2fe-7402-11ef-bceb-c7b262605968@mon.smithi144.service' 2024-09-16T08:28:33.476 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-16T08:28:33.476 INFO:tasks.cephadm.mon.smithi144:Stopped mon.smithi144 2024-09-16T08:28:33.476 INFO:tasks.cephadm.osd.0:Stopping osd.0... 2024-09-16T08:28:33.476 DEBUG:teuthology.orchestra.run.smithi028:> sudo systemctl stop ceph-5622b2fe-7402-11ef-bceb-c7b262605968@osd.0 2024-09-16T08:28:33.797 INFO:journalctl@ceph.osd.0.smithi028.stdout:Sep 16 08:28:33 smithi028 systemd[1]: Stopping Ceph osd.0 for 5622b2fe-7402-11ef-bceb-c7b262605968... 2024-09-16T08:28:34.197 INFO:journalctl@ceph.osd.0.smithi028.stdout:Sep 16 08:28:33 smithi028 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-0[68905]: 2024-09-16T08:28:33.797+0000 7f0ffbedf640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-16T08:28:34.197 INFO:journalctl@ceph.osd.0.smithi028.stdout:Sep 16 08:28:33 smithi028 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-0[68905]: 2024-09-16T08:28:33.797+0000 7f0ffbedf640 -1 osd.0 158 *** Got signal Terminated *** 2024-09-16T08:28:34.197 INFO:journalctl@ceph.osd.0.smithi028.stdout:Sep 16 08:28:33 smithi028 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-0[68905]: 2024-09-16T08:28:33.797+0000 7f0ffbedf640 -1 osd.0 158 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-16T08:28:39.697 INFO:journalctl@ceph.osd.0.smithi028.stdout:Sep 16 08:28:39 smithi028 podman[94243]: 2024-09-16 08:28:39.401014611 +0000 UTC m=+5.712815832 container died 2fc02ba4f14f01fea22c133a927dc68cf30104c78c62455e439e3f26bb320214 (image=quay.ceph.io/ceph-ci/ceph@sha256:f7ad55f9ccd17ba66aadb19b4d291e67b60ea24e5d8a0c0e92941fd59c81e4ce, name=ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-0, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, RELEASE=wip-guits-main-2024-09-13-1248-8293d73, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, CEPH_POINT_RELEASE=, ceph=True) 2024-09-16T08:28:40.197 INFO:journalctl@ceph.osd.0.smithi028.stdout:Sep 16 08:28:39 smithi028 podman[94243]: 2024-09-16 08:28:39.724719875 +0000 UTC m=+6.036521095 container cleanup 2fc02ba4f14f01fea22c133a927dc68cf30104c78c62455e439e3f26bb320214 (image=quay.ceph.io/ceph-ci/ceph@sha256:f7ad55f9ccd17ba66aadb19b4d291e67b60ea24e5d8a0c0e92941fd59c81e4ce, name=ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-0, RELEASE=wip-guits-main-2024-09-13-1248-8293d73, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, io.buildah.version=1.37.2) 2024-09-16T08:28:40.197 INFO:journalctl@ceph.osd.0.smithi028.stdout:Sep 16 08:28:39 smithi028 bash[94243]: ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-0 2024-09-16T08:28:40.197 INFO:journalctl@ceph.osd.0.smithi028.stdout:Sep 16 08:28:39 smithi028 podman[94276]: 2024-09-16 08:28:39.956840648 +0000 UTC m=+0.558102385 container remove 2fc02ba4f14f01fea22c133a927dc68cf30104c78c62455e439e3f26bb320214 (image=quay.ceph.io/ceph-ci/ceph@sha256:f7ad55f9ccd17ba66aadb19b4d291e67b60ea24e5d8a0c0e92941fd59c81e4ce, name=ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-0, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, GIT_CLEAN=True, ceph=True, RELEASE=wip-guits-main-2024-09-13-1248-8293d73, org.label-schema.schema-version=1.0) 2024-09-16T08:28:40.697 INFO:journalctl@ceph.osd.0.smithi028.stdout:Sep 16 08:28:40 smithi028 podman[94363]: 2024-09-16 08:28:40.40790991 +0000 UTC m=+0.107374033 container create 80d5335cc66874be927e83c8c11140050c5a1e63f6a2e298bf5fc5d8285a1f5f (image=quay.ceph.io/ceph-ci/ceph@sha256:f7ad55f9ccd17ba66aadb19b4d291e67b60ea24e5d8a0c0e92941fd59c81e4ce, name=ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-0-deactivate, GIT_BRANCH=HEAD, RELEASE=wip-guits-main-2024-09-13-1248-8293d73, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965) 2024-09-16T08:28:40.697 INFO:journalctl@ceph.osd.0.smithi028.stdout:Sep 16 08:28:40 smithi028 podman[94363]: 2024-09-16 08:28:40.317694513 +0000 UTC m=+0.017158636 image pull 425e316ba8ebf74f04aa3640f7deb4fa3f293a78d0c8d097cb679eaf86ddca6c quay.ceph.io/ceph-ci/ceph@sha256:f7ad55f9ccd17ba66aadb19b4d291e67b60ea24e5d8a0c0e92941fd59c81e4ce 2024-09-16T08:28:40.697 INFO:journalctl@ceph.osd.0.smithi028.stdout:Sep 16 08:28:40 smithi028 podman[94363]: 2024-09-16 08:28:40.643765204 +0000 UTC m=+0.343229327 container init 80d5335cc66874be927e83c8c11140050c5a1e63f6a2e298bf5fc5d8285a1f5f (image=quay.ceph.io/ceph-ci/ceph@sha256:f7ad55f9ccd17ba66aadb19b4d291e67b60ea24e5d8a0c0e92941fd59c81e4ce, name=ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-0-deactivate, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-guits-main-2024-09-13-1248-8293d73, ceph=True, org.label-schema.license=GPLv2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=) 2024-09-16T08:28:40.697 INFO:journalctl@ceph.osd.0.smithi028.stdout:Sep 16 08:28:40 smithi028 podman[94363]: 2024-09-16 08:28:40.647484702 +0000 UTC m=+0.346948831 container start 80d5335cc66874be927e83c8c11140050c5a1e63f6a2e298bf5fc5d8285a1f5f (image=quay.ceph.io/ceph-ci/ceph@sha256:f7ad55f9ccd17ba66aadb19b4d291e67b60ea24e5d8a0c0e92941fd59c81e4ce, name=ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-0-deactivate, io.buildah.version=1.37.2, RELEASE=wip-guits-main-2024-09-13-1248-8293d73, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, ceph=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2) 2024-09-16T08:28:41.156 INFO:journalctl@ceph.osd.0.smithi028.stdout:Sep 16 08:28:40 smithi028 podman[94363]: 2024-09-16 08:28:40.743871005 +0000 UTC m=+0.443335139 container attach 80d5335cc66874be927e83c8c11140050c5a1e63f6a2e298bf5fc5d8285a1f5f (image=quay.ceph.io/ceph-ci/ceph@sha256:f7ad55f9ccd17ba66aadb19b4d291e67b60ea24e5d8a0c0e92941fd59c81e4ce, name=ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-0-deactivate, GIT_BRANCH=HEAD, ceph=True, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , RELEASE=wip-guits-main-2024-09-13-1248-8293d73, org.label-schema.build-date=20240731) 2024-09-16T08:28:41.157 INFO:journalctl@ceph.osd.0.smithi028.stdout:Sep 16 08:28:40 smithi028 podman[94363]: 2024-09-16 08:28:40.845524513 +0000 UTC m=+0.544988639 container died 80d5335cc66874be927e83c8c11140050c5a1e63f6a2e298bf5fc5d8285a1f5f (image=quay.ceph.io/ceph-ci/ceph@sha256:f7ad55f9ccd17ba66aadb19b4d291e67b60ea24e5d8a0c0e92941fd59c81e4ce, name=ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-0-deactivate, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, RELEASE=wip-guits-main-2024-09-13-1248-8293d73, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, ceph=True, maintainer=Guillaume Abrioux , GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-16T08:28:41.452 DEBUG:teuthology.orchestra.run.smithi028:> sudo pkill -f 'journalctl -f -n 0 -u ceph-5622b2fe-7402-11ef-bceb-c7b262605968@osd.0.service' 2024-09-16T08:28:41.494 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-16T08:28:41.495 INFO:tasks.cephadm.osd.0:Stopped osd.0 2024-09-16T08:28:41.495 INFO:tasks.cephadm.osd.1:Stopping osd.1... 2024-09-16T08:28:41.495 DEBUG:teuthology.orchestra.run.smithi028:> sudo systemctl stop ceph-5622b2fe-7402-11ef-bceb-c7b262605968@osd.1 2024-09-16T08:28:41.697 INFO:journalctl@ceph.osd.1.smithi028.stdout:Sep 16 08:28:41 smithi028 systemd[1]: Stopping Ceph osd.1 for 5622b2fe-7402-11ef-bceb-c7b262605968... 2024-09-16T08:28:42.197 INFO:journalctl@ceph.osd.1.smithi028.stdout:Sep 16 08:28:41 smithi028 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-1[71361]: 2024-09-16T08:28:41.842+0000 7f6be092a640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.1 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-16T08:28:42.197 INFO:journalctl@ceph.osd.1.smithi028.stdout:Sep 16 08:28:41 smithi028 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-1[71361]: 2024-09-16T08:28:41.842+0000 7f6be092a640 -1 osd.1 158 *** Got signal Terminated *** 2024-09-16T08:28:42.197 INFO:journalctl@ceph.osd.1.smithi028.stdout:Sep 16 08:28:41 smithi028 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-1[71361]: 2024-09-16T08:28:41.842+0000 7f6be092a640 -1 osd.1 158 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-16T08:28:47.643 INFO:journalctl@ceph.osd.1.smithi028.stdout:Sep 16 08:28:47 smithi028 podman[94468]: 2024-09-16 08:28:47.383927396 +0000 UTC m=+5.657539006 container died df34bd603d8f561da8f37f5dc13a453f49ffdedf58bdeb0658d1cca697dae0f2 (image=quay.ceph.io/ceph-ci/ceph@sha256:f7ad55f9ccd17ba66aadb19b4d291e67b60ea24e5d8a0c0e92941fd59c81e4ce, name=ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-1, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, RELEASE=wip-guits-main-2024-09-13-1248-8293d73, ceph=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_CLEAN=True) 2024-09-16T08:28:47.947 INFO:journalctl@ceph.osd.1.smithi028.stdout:Sep 16 08:28:47 smithi028 podman[94468]: 2024-09-16 08:28:47.733927613 +0000 UTC m=+6.007539209 container cleanup df34bd603d8f561da8f37f5dc13a453f49ffdedf58bdeb0658d1cca697dae0f2 (image=quay.ceph.io/ceph-ci/ceph@sha256:f7ad55f9ccd17ba66aadb19b4d291e67b60ea24e5d8a0c0e92941fd59c81e4ce, name=ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-1, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, RELEASE=wip-guits-main-2024-09-13-1248-8293d73, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_CLEAN=True, ceph=True, org.label-schema.license=GPLv2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965) 2024-09-16T08:28:47.947 INFO:journalctl@ceph.osd.1.smithi028.stdout:Sep 16 08:28:47 smithi028 bash[94468]: ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-1 2024-09-16T08:28:48.352 INFO:journalctl@ceph.osd.1.smithi028.stdout:Sep 16 08:28:48 smithi028 podman[94487]: 2024-09-16 08:28:48.001890687 +0000 UTC m=+0.614230684 container remove df34bd603d8f561da8f37f5dc13a453f49ffdedf58bdeb0658d1cca697dae0f2 (image=quay.ceph.io/ceph-ci/ceph@sha256:f7ad55f9ccd17ba66aadb19b4d291e67b60ea24e5d8a0c0e92941fd59c81e4ce, name=ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-1, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-guits-main-2024-09-13-1248-8293d73, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, ceph=True, CEPH_POINT_RELEASE=) 2024-09-16T08:28:48.638 INFO:journalctl@ceph.osd.1.smithi028.stdout:Sep 16 08:28:48 smithi028 podman[94573]: 2024-09-16 08:28:48.35282077 +0000 UTC m=+0.093585222 container create 532aee46a30f6769296204b2d878d6b09c9ee207d0125c5f77c4fc51771fcd75 (image=quay.ceph.io/ceph-ci/ceph@sha256:f7ad55f9ccd17ba66aadb19b4d291e67b60ea24e5d8a0c0e92941fd59c81e4ce, name=ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-1-deactivate, org.label-schema.schema-version=1.0, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, ceph=True, maintainer=Guillaume Abrioux , RELEASE=wip-guits-main-2024-09-13-1248-8293d73, io.buildah.version=1.37.2) 2024-09-16T08:28:48.638 INFO:journalctl@ceph.osd.1.smithi028.stdout:Sep 16 08:28:48 smithi028 podman[94573]: 2024-09-16 08:28:48.276232383 +0000 UTC m=+0.016996806 image pull 425e316ba8ebf74f04aa3640f7deb4fa3f293a78d0c8d097cb679eaf86ddca6c quay.ceph.io/ceph-ci/ceph@sha256:f7ad55f9ccd17ba66aadb19b4d291e67b60ea24e5d8a0c0e92941fd59c81e4ce 2024-09-16T08:28:48.638 INFO:journalctl@ceph.osd.1.smithi028.stdout:Sep 16 08:28:48 smithi028 podman[94573]: 2024-09-16 08:28:48.563368185 +0000 UTC m=+0.304132609 container init 532aee46a30f6769296204b2d878d6b09c9ee207d0125c5f77c4fc51771fcd75 (image=quay.ceph.io/ceph-ci/ceph@sha256:f7ad55f9ccd17ba66aadb19b4d291e67b60ea24e5d8a0c0e92941fd59c81e4ce, name=ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-1-deactivate, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, ceph=True, org.label-schema.schema-version=1.0, RELEASE=wip-guits-main-2024-09-13-1248-8293d73, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_CLEAN=True) 2024-09-16T08:28:48.638 INFO:journalctl@ceph.osd.1.smithi028.stdout:Sep 16 08:28:48 smithi028 podman[94573]: 2024-09-16 08:28:48.567326908 +0000 UTC m=+0.308091323 container start 532aee46a30f6769296204b2d878d6b09c9ee207d0125c5f77c4fc51771fcd75 (image=quay.ceph.io/ceph-ci/ceph@sha256:f7ad55f9ccd17ba66aadb19b4d291e67b60ea24e5d8a0c0e92941fd59c81e4ce, name=ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-1-deactivate, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_CLEAN=True, RELEASE=wip-guits-main-2024-09-13-1248-8293d73, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0) 2024-09-16T08:28:48.638 INFO:journalctl@ceph.osd.1.smithi028.stdout:Sep 16 08:28:48 smithi028 podman[94573]: 2024-09-16 08:28:48.638239008 +0000 UTC m=+0.379003428 container attach 532aee46a30f6769296204b2d878d6b09c9ee207d0125c5f77c4fc51771fcd75 (image=quay.ceph.io/ceph-ci/ceph@sha256:f7ad55f9ccd17ba66aadb19b4d291e67b60ea24e5d8a0c0e92941fd59c81e4ce, name=ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-1-deactivate, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, CEPH_POINT_RELEASE=, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_BRANCH=HEAD, RELEASE=wip-guits-main-2024-09-13-1248-8293d73) 2024-09-16T08:28:48.947 INFO:journalctl@ceph.osd.1.smithi028.stdout:Sep 16 08:28:48 smithi028 podman[94573]: 2024-09-16 08:28:48.759609096 +0000 UTC m=+0.500373513 container died 532aee46a30f6769296204b2d878d6b09c9ee207d0125c5f77c4fc51771fcd75 (image=quay.ceph.io/ceph-ci/ceph@sha256:f7ad55f9ccd17ba66aadb19b4d291e67b60ea24e5d8a0c0e92941fd59c81e4ce, name=ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-1-deactivate, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, ceph=True, CEPH_POINT_RELEASE=, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, RELEASE=wip-guits-main-2024-09-13-1248-8293d73) 2024-09-16T08:28:49.387 DEBUG:teuthology.orchestra.run.smithi028:> sudo pkill -f 'journalctl -f -n 0 -u ceph-5622b2fe-7402-11ef-bceb-c7b262605968@osd.1.service' 2024-09-16T08:28:49.428 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-16T08:28:49.429 INFO:tasks.cephadm.osd.1:Stopped osd.1 2024-09-16T08:28:49.429 INFO:tasks.cephadm.osd.2:Stopping osd.2... 2024-09-16T08:28:49.429 DEBUG:teuthology.orchestra.run.smithi028:> sudo systemctl stop ceph-5622b2fe-7402-11ef-bceb-c7b262605968@osd.2 2024-09-16T08:28:49.697 INFO:journalctl@ceph.osd.2.smithi028.stdout:Sep 16 08:28:49 smithi028 systemd[1]: Stopping Ceph osd.2 for 5622b2fe-7402-11ef-bceb-c7b262605968... 2024-09-16T08:28:50.197 INFO:journalctl@ceph.osd.2.smithi028.stdout:Sep 16 08:28:49 smithi028 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-2[73744]: 2024-09-16T08:28:49.779+0000 7fe352f98640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.2 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-16T08:28:50.197 INFO:journalctl@ceph.osd.2.smithi028.stdout:Sep 16 08:28:49 smithi028 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-2[73744]: 2024-09-16T08:28:49.779+0000 7fe352f98640 -1 osd.2 158 *** Got signal Terminated *** 2024-09-16T08:28:50.197 INFO:journalctl@ceph.osd.2.smithi028.stdout:Sep 16 08:28:49 smithi028 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-2[73744]: 2024-09-16T08:28:49.779+0000 7fe352f98640 -1 osd.2 158 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-16T08:28:55.513 INFO:journalctl@ceph.osd.2.smithi028.stdout:Sep 16 08:28:55 smithi028 podman[94675]: 2024-09-16 08:28:55.222192578 +0000 UTC m=+5.558950774 container died c92be6327b129d0890c6956a6187445ba7b46761ace25648ea7e30a60fd67da8 (image=quay.ceph.io/ceph-ci/ceph@sha256:f7ad55f9ccd17ba66aadb19b4d291e67b60ea24e5d8a0c0e92941fd59c81e4ce, name=ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-2, RELEASE=wip-guits-main-2024-09-13-1248-8293d73, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_CLEAN=True, ceph=True, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-16T08:28:55.846 INFO:journalctl@ceph.osd.2.smithi028.stdout:Sep 16 08:28:55 smithi028 podman[94675]: 2024-09-16 08:28:55.612045466 +0000 UTC m=+5.948803661 container cleanup c92be6327b129d0890c6956a6187445ba7b46761ace25648ea7e30a60fd67da8 (image=quay.ceph.io/ceph-ci/ceph@sha256:f7ad55f9ccd17ba66aadb19b4d291e67b60ea24e5d8a0c0e92941fd59c81e4ce, name=ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, RELEASE=wip-guits-main-2024-09-13-1248-8293d73, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS) 2024-09-16T08:28:55.846 INFO:journalctl@ceph.osd.2.smithi028.stdout:Sep 16 08:28:55 smithi028 bash[94675]: ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-2 2024-09-16T08:28:56.155 INFO:journalctl@ceph.osd.2.smithi028.stdout:Sep 16 08:28:55 smithi028 podman[94690]: 2024-09-16 08:28:55.846536987 +0000 UTC m=+0.619199689 container remove c92be6327b129d0890c6956a6187445ba7b46761ace25648ea7e30a60fd67da8 (image=quay.ceph.io/ceph-ci/ceph@sha256:f7ad55f9ccd17ba66aadb19b4d291e67b60ea24e5d8a0c0e92941fd59c81e4ce, name=ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-2, GIT_BRANCH=HEAD, ceph=True, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, RELEASE=wip-guits-main-2024-09-13-1248-8293d73, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2) 2024-09-16T08:28:56.447 INFO:journalctl@ceph.osd.2.smithi028.stdout:Sep 16 08:28:56 smithi028 podman[94777]: 2024-09-16 08:28:56.155675812 +0000 UTC m=+0.096268503 container create 05c89c9db5a150e30a8823161ed2beacfa2d85cd2c8d286defd2f98427accbaf (image=quay.ceph.io/ceph-ci/ceph@sha256:f7ad55f9ccd17ba66aadb19b4d291e67b60ea24e5d8a0c0e92941fd59c81e4ce, name=ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-2-deactivate, RELEASE=wip-guits-main-2024-09-13-1248-8293d73, GIT_CLEAN=True, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, io.buildah.version=1.37.2) 2024-09-16T08:28:56.448 INFO:journalctl@ceph.osd.2.smithi028.stdout:Sep 16 08:28:56 smithi028 podman[94777]: 2024-09-16 08:28:56.076265407 +0000 UTC m=+0.016858073 image pull 425e316ba8ebf74f04aa3640f7deb4fa3f293a78d0c8d097cb679eaf86ddca6c quay.ceph.io/ceph-ci/ceph@sha256:f7ad55f9ccd17ba66aadb19b4d291e67b60ea24e5d8a0c0e92941fd59c81e4ce 2024-09-16T08:28:56.448 INFO:journalctl@ceph.osd.2.smithi028.stdout:Sep 16 08:28:56 smithi028 podman[94777]: 2024-09-16 08:28:56.391462103 +0000 UTC m=+0.332054753 container init 05c89c9db5a150e30a8823161ed2beacfa2d85cd2c8d286defd2f98427accbaf (image=quay.ceph.io/ceph-ci/ceph@sha256:f7ad55f9ccd17ba66aadb19b4d291e67b60ea24e5d8a0c0e92941fd59c81e4ce, name=ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-2-deactivate, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.license=GPLv2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, RELEASE=wip-guits-main-2024-09-13-1248-8293d73, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-16T08:28:56.448 INFO:journalctl@ceph.osd.2.smithi028.stdout:Sep 16 08:28:56 smithi028 podman[94777]: 2024-09-16 08:28:56.394944814 +0000 UTC m=+0.335537471 container start 05c89c9db5a150e30a8823161ed2beacfa2d85cd2c8d286defd2f98427accbaf (image=quay.ceph.io/ceph-ci/ceph@sha256:f7ad55f9ccd17ba66aadb19b4d291e67b60ea24e5d8a0c0e92941fd59c81e4ce, name=ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-2-deactivate, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_CLEAN=True, CEPH_POINT_RELEASE=, RELEASE=wip-guits-main-2024-09-13-1248-8293d73, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2) 2024-09-16T08:28:56.887 INFO:journalctl@ceph.osd.2.smithi028.stdout:Sep 16 08:28:56 smithi028 podman[94777]: 2024-09-16 08:28:56.49163117 +0000 UTC m=+0.432223822 container attach 05c89c9db5a150e30a8823161ed2beacfa2d85cd2c8d286defd2f98427accbaf (image=quay.ceph.io/ceph-ci/ceph@sha256:f7ad55f9ccd17ba66aadb19b4d291e67b60ea24e5d8a0c0e92941fd59c81e4ce, name=ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-2-deactivate, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, RELEASE=wip-guits-main-2024-09-13-1248-8293d73, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, ceph=True, GIT_CLEAN=True) 2024-09-16T08:28:56.887 INFO:journalctl@ceph.osd.2.smithi028.stdout:Sep 16 08:28:56 smithi028 podman[94777]: 2024-09-16 08:28:56.597409542 +0000 UTC m=+0.538002201 container died 05c89c9db5a150e30a8823161ed2beacfa2d85cd2c8d286defd2f98427accbaf (image=quay.ceph.io/ceph-ci/ceph@sha256:f7ad55f9ccd17ba66aadb19b4d291e67b60ea24e5d8a0c0e92941fd59c81e4ce, name=ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-2-deactivate, org.label-schema.license=GPLv2, RELEASE=wip-guits-main-2024-09-13-1248-8293d73, ceph=True, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-16T08:28:57.140 INFO:journalctl@ceph.osd.2.smithi028.stdout:Sep 16 08:28:57 smithi028 podman[94800]: 2024-09-16 08:28:57.136739372 +0000 UTC m=+0.530979957 container remove 05c89c9db5a150e30a8823161ed2beacfa2d85cd2c8d286defd2f98427accbaf (image=quay.ceph.io/ceph-ci/ceph@sha256:f7ad55f9ccd17ba66aadb19b4d291e67b60ea24e5d8a0c0e92941fd59c81e4ce, name=ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-2-deactivate, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, RELEASE=wip-guits-main-2024-09-13-1248-8293d73, ceph=True, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_CLEAN=True, maintainer=Guillaume Abrioux ) 2024-09-16T08:28:57.169 DEBUG:teuthology.orchestra.run.smithi028:> sudo pkill -f 'journalctl -f -n 0 -u ceph-5622b2fe-7402-11ef-bceb-c7b262605968@osd.2.service' 2024-09-16T08:28:57.213 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-16T08:28:57.213 INFO:tasks.cephadm.osd.2:Stopped osd.2 2024-09-16T08:28:57.213 INFO:tasks.cephadm.osd.3:Stopping osd.3... 2024-09-16T08:28:57.213 DEBUG:teuthology.orchestra.run.smithi028:> sudo systemctl stop ceph-5622b2fe-7402-11ef-bceb-c7b262605968@osd.3 2024-09-16T08:28:57.447 INFO:journalctl@ceph.osd.3.smithi028.stdout:Sep 16 08:28:57 smithi028 systemd[1]: Stopping Ceph osd.3 for 5622b2fe-7402-11ef-bceb-c7b262605968... 2024-09-16T08:28:57.947 INFO:journalctl@ceph.osd.3.smithi028.stdout:Sep 16 08:28:57 smithi028 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-3[76241]: 2024-09-16T08:28:57.664+0000 7fd98daf6640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.3 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-16T08:28:57.948 INFO:journalctl@ceph.osd.3.smithi028.stdout:Sep 16 08:28:57 smithi028 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-3[76241]: 2024-09-16T08:28:57.664+0000 7fd98daf6640 -1 osd.3 158 *** Got signal Terminated *** 2024-09-16T08:28:57.948 INFO:journalctl@ceph.osd.3.smithi028.stdout:Sep 16 08:28:57 smithi028 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-3[76241]: 2024-09-16T08:28:57.664+0000 7fd98daf6640 -1 osd.3 158 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-16T08:29:03.332 INFO:journalctl@ceph.osd.5.smithi144.stdout:Sep 16 08:29:02 smithi144 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-5[55067]: 2024-09-16T08:29:02.896+0000 7fdc67dd8640 -1 osd.5 158 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-09-16T08:28:37.925923+0000 front 2024-09-16T08:28:37.925909+0000 (oldest deadline 2024-09-16T08:29:02.625815+0000) 2024-09-16T08:29:03.423 INFO:journalctl@ceph.osd.3.smithi028.stdout:Sep 16 08:29:03 smithi028 podman[94878]: 2024-09-16 08:29:03.156611777 +0000 UTC m=+5.708001240 container died bd84ba1224f2d5e5f2a6a62fb7ed5cb90fbe74581f2807beaa2f9af07a91afb2 (image=quay.ceph.io/ceph-ci/ceph@sha256:f7ad55f9ccd17ba66aadb19b4d291e67b60ea24e5d8a0c0e92941fd59c81e4ce, name=ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-3, RELEASE=wip-guits-main-2024-09-13-1248-8293d73, org.label-schema.vendor=CentOS, ceph=True, maintainer=Guillaume Abrioux , GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731) 2024-09-16T08:29:03.697 INFO:journalctl@ceph.osd.3.smithi028.stdout:Sep 16 08:29:03 smithi028 podman[94878]: 2024-09-16 08:29:03.522978434 +0000 UTC m=+6.074367900 container cleanup bd84ba1224f2d5e5f2a6a62fb7ed5cb90fbe74581f2807beaa2f9af07a91afb2 (image=quay.ceph.io/ceph-ci/ceph@sha256:f7ad55f9ccd17ba66aadb19b4d291e67b60ea24e5d8a0c0e92941fd59c81e4ce, name=ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-3, ceph=True, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_CLEAN=True, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-guits-main-2024-09-13-1248-8293d73, CEPH_POINT_RELEASE=) 2024-09-16T08:29:03.697 INFO:journalctl@ceph.osd.3.smithi028.stdout:Sep 16 08:29:03 smithi028 bash[94878]: ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-3 2024-09-16T08:29:04.039 INFO:journalctl@ceph.osd.5.smithi144.stdout:Sep 16 08:29:03 smithi144 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-5[55067]: 2024-09-16T08:29:03.900+0000 7fdc67dd8640 -1 osd.5 158 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-09-16T08:28:37.925923+0000 front 2024-09-16T08:28:37.925909+0000 (oldest deadline 2024-09-16T08:29:02.625815+0000) 2024-09-16T08:29:04.040 INFO:journalctl@ceph.osd.6.smithi144.stdout:Sep 16 08:29:03 smithi144 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-6[57105]: 2024-09-16T08:29:03.665+0000 7fc05286c640 -1 osd.6 158 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-09-16T08:28:38.187159+0000 front 2024-09-16T08:28:38.187196+0000 (oldest deadline 2024-09-16T08:29:02.887071+0000) 2024-09-16T08:29:04.088 INFO:journalctl@ceph.osd.3.smithi028.stdout:Sep 16 08:29:03 smithi028 podman[94910]: 2024-09-16 08:29:03.757394714 +0000 UTC m=+0.601890943 container remove bd84ba1224f2d5e5f2a6a62fb7ed5cb90fbe74581f2807beaa2f9af07a91afb2 (image=quay.ceph.io/ceph-ci/ceph@sha256:f7ad55f9ccd17ba66aadb19b4d291e67b60ea24e5d8a0c0e92941fd59c81e4ce, name=ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-3, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, org.label-schema.schema-version=1.0, RELEASE=wip-guits-main-2024-09-13-1248-8293d73, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2) 2024-09-16T08:29:04.332 INFO:journalctl@ceph.osd.7.smithi144.stdout:Sep 16 08:29:04 smithi144 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-7[59146]: 2024-09-16T08:29:04.208+0000 7f4a9aa4d640 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-09-16T08:28:37.859409+0000 front 2024-09-16T08:28:37.859423+0000 (oldest deadline 2024-09-16T08:29:03.759284+0000) 2024-09-16T08:29:04.332 INFO:journalctl@ceph.osd.4.smithi144.stdout:Sep 16 08:29:04 smithi144 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-4[53035]: 2024-09-16T08:29:04.039+0000 7f3052327640 -1 osd.4 158 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-09-16T08:28:37.576788+0000 front 2024-09-16T08:28:37.576964+0000 (oldest deadline 2024-09-16T08:29:03.476816+0000) 2024-09-16T08:29:04.385 INFO:journalctl@ceph.osd.3.smithi028.stdout:Sep 16 08:29:04 smithi028 podman[95000]: 2024-09-16 08:29:03.992224827 +0000 UTC m=+0.016950367 image pull 425e316ba8ebf74f04aa3640f7deb4fa3f293a78d0c8d097cb679eaf86ddca6c quay.ceph.io/ceph-ci/ceph@sha256:f7ad55f9ccd17ba66aadb19b4d291e67b60ea24e5d8a0c0e92941fd59c81e4ce 2024-09-16T08:29:04.385 INFO:journalctl@ceph.osd.3.smithi028.stdout:Sep 16 08:29:04 smithi028 podman[95000]: 2024-09-16 08:29:04.091354447 +0000 UTC m=+0.116079986 container create d8306f06699e392b37c1d535c287a6e391b171b97a2c1e100c4120bed770db14 (image=quay.ceph.io/ceph-ci/ceph@sha256:f7ad55f9ccd17ba66aadb19b4d291e67b60ea24e5d8a0c0e92941fd59c81e4ce, name=ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-3-deactivate, org.label-schema.vendor=CentOS, RELEASE=wip-guits-main-2024-09-13-1248-8293d73, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.license=GPLv2, ceph=True, GIT_CLEAN=True, org.label-schema.schema-version=1.0) 2024-09-16T08:29:04.385 INFO:journalctl@ceph.osd.3.smithi028.stdout:Sep 16 08:29:04 smithi028 podman[95000]: 2024-09-16 08:29:04.285398538 +0000 UTC m=+0.310124091 container init d8306f06699e392b37c1d535c287a6e391b171b97a2c1e100c4120bed770db14 (image=quay.ceph.io/ceph-ci/ceph@sha256:f7ad55f9ccd17ba66aadb19b4d291e67b60ea24e5d8a0c0e92941fd59c81e4ce, name=ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-3-deactivate, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, ceph=True, org.label-schema.schema-version=1.0, RELEASE=wip-guits-main-2024-09-13-1248-8293d73, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=) 2024-09-16T08:29:04.385 INFO:journalctl@ceph.osd.3.smithi028.stdout:Sep 16 08:29:04 smithi028 podman[95000]: 2024-09-16 08:29:04.28892688 +0000 UTC m=+0.313652421 container start d8306f06699e392b37c1d535c287a6e391b171b97a2c1e100c4120bed770db14 (image=quay.ceph.io/ceph-ci/ceph@sha256:f7ad55f9ccd17ba66aadb19b4d291e67b60ea24e5d8a0c0e92941fd59c81e4ce, name=ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-3-deactivate, io.buildah.version=1.37.2, GIT_CLEAN=True, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=wip-guits-main-2024-09-13-1248-8293d73, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=) 2024-09-16T08:29:04.385 INFO:journalctl@ceph.osd.3.smithi028.stdout:Sep 16 08:29:04 smithi028 podman[95000]: 2024-09-16 08:29:04.385274805 +0000 UTC m=+0.410000361 container attach d8306f06699e392b37c1d535c287a6e391b171b97a2c1e100c4120bed770db14 (image=quay.ceph.io/ceph-ci/ceph@sha256:f7ad55f9ccd17ba66aadb19b4d291e67b60ea24e5d8a0c0e92941fd59c81e4ce, name=ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-3-deactivate, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, RELEASE=wip-guits-main-2024-09-13-1248-8293d73, GIT_CLEAN=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, ceph=True, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux ) 2024-09-16T08:29:04.697 INFO:journalctl@ceph.osd.3.smithi028.stdout:Sep 16 08:29:04 smithi028 podman[95000]: 2024-09-16 08:29:04.483529195 +0000 UTC m=+0.508254736 container died d8306f06699e392b37c1d535c287a6e391b171b97a2c1e100c4120bed770db14 (image=quay.ceph.io/ceph-ci/ceph@sha256:f7ad55f9ccd17ba66aadb19b4d291e67b60ea24e5d8a0c0e92941fd59c81e4ce, name=ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-3-deactivate, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, RELEASE=wip-guits-main-2024-09-13-1248-8293d73, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, ceph=True, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-16T08:29:04.914 INFO:journalctl@ceph.osd.6.smithi144.stdout:Sep 16 08:29:04 smithi144 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-6[57105]: 2024-09-16T08:29:04.647+0000 7fc05286c640 -1 osd.6 158 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-09-16T08:28:38.187159+0000 front 2024-09-16T08:28:38.187196+0000 (oldest deadline 2024-09-16T08:29:02.887071+0000) 2024-09-16T08:29:05.033 INFO:journalctl@ceph.osd.3.smithi028.stdout:Sep 16 08:29:05 smithi028 podman[95023]: 2024-09-16 08:29:05.030673081 +0000 UTC m=+0.540619589 container remove d8306f06699e392b37c1d535c287a6e391b171b97a2c1e100c4120bed770db14 (image=quay.ceph.io/ceph-ci/ceph@sha256:f7ad55f9ccd17ba66aadb19b4d291e67b60ea24e5d8a0c0e92941fd59c81e4ce, name=ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-3-deactivate, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, ceph=True, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-guits-main-2024-09-13-1248-8293d73, org.label-schema.vendor=CentOS, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux ) 2024-09-16T08:29:05.056 DEBUG:teuthology.orchestra.run.smithi028:> sudo pkill -f 'journalctl -f -n 0 -u ceph-5622b2fe-7402-11ef-bceb-c7b262605968@osd.3.service' 2024-09-16T08:29:05.101 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-16T08:29:05.101 INFO:tasks.cephadm.osd.3:Stopped osd.3 2024-09-16T08:29:05.101 INFO:tasks.cephadm.osd.4:Stopping osd.4... 2024-09-16T08:29:05.101 DEBUG:teuthology.orchestra.run.smithi144:> sudo systemctl stop ceph-5622b2fe-7402-11ef-bceb-c7b262605968@osd.4 2024-09-16T08:29:05.243 INFO:journalctl@ceph.osd.5.smithi144.stdout:Sep 16 08:29:04 smithi144 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-5[55067]: 2024-09-16T08:29:04.914+0000 7fdc67dd8640 -1 osd.5 158 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-09-16T08:28:37.925923+0000 front 2024-09-16T08:28:37.925909+0000 (oldest deadline 2024-09-16T08:29:02.625815+0000) 2024-09-16T08:29:05.244 INFO:journalctl@ceph.osd.4.smithi144.stdout:Sep 16 08:29:04 smithi144 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-4[53035]: 2024-09-16T08:29:04.992+0000 7f3052327640 -1 osd.4 158 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-09-16T08:28:37.576788+0000 front 2024-09-16T08:28:37.576964+0000 (oldest deadline 2024-09-16T08:29:03.476816+0000) 2024-09-16T08:29:05.244 INFO:journalctl@ceph.osd.4.smithi144.stdout:Sep 16 08:29:05 smithi144 systemd[1]: Stopping Ceph osd.4 for 5622b2fe-7402-11ef-bceb-c7b262605968... 2024-09-16T08:29:05.582 INFO:journalctl@ceph.osd.7.smithi144.stdout:Sep 16 08:29:05 smithi144 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-7[59146]: 2024-09-16T08:29:05.242+0000 7f4a9aa4d640 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-09-16T08:28:37.859409+0000 front 2024-09-16T08:28:37.859423+0000 (oldest deadline 2024-09-16T08:29:03.759284+0000) 2024-09-16T08:29:05.582 INFO:journalctl@ceph.osd.4.smithi144.stdout:Sep 16 08:29:05 smithi144 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-4[53035]: 2024-09-16T08:29:05.429+0000 7f3059d1a640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.4 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-16T08:29:05.582 INFO:journalctl@ceph.osd.4.smithi144.stdout:Sep 16 08:29:05 smithi144 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-4[53035]: 2024-09-16T08:29:05.429+0000 7f3059d1a640 -1 osd.4 158 *** Got signal Terminated *** 2024-09-16T08:29:05.582 INFO:journalctl@ceph.osd.4.smithi144.stdout:Sep 16 08:29:05 smithi144 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-4[53035]: 2024-09-16T08:29:05.429+0000 7f3059d1a640 -1 osd.4 158 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-16T08:29:05.943 INFO:journalctl@ceph.osd.6.smithi144.stdout:Sep 16 08:29:05 smithi144 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-6[57105]: 2024-09-16T08:29:05.685+0000 7fc05286c640 -1 osd.6 158 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-09-16T08:28:38.187159+0000 front 2024-09-16T08:28:38.187196+0000 (oldest deadline 2024-09-16T08:29:02.887071+0000) 2024-09-16T08:29:05.943 INFO:journalctl@ceph.osd.5.smithi144.stdout:Sep 16 08:29:05 smithi144 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-5[55067]: 2024-09-16T08:29:05.881+0000 7fdc67dd8640 -1 osd.5 158 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-09-16T08:28:37.925923+0000 front 2024-09-16T08:28:37.925909+0000 (oldest deadline 2024-09-16T08:29:02.625815+0000) 2024-09-16T08:29:06.266 INFO:journalctl@ceph.osd.4.smithi144.stdout:Sep 16 08:29:05 smithi144 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-4[53035]: 2024-09-16T08:29:05.943+0000 7f3052327640 -1 osd.4 158 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-09-16T08:28:37.576788+0000 front 2024-09-16T08:28:37.576964+0000 (oldest deadline 2024-09-16T08:29:03.476816+0000) 2024-09-16T08:29:06.582 INFO:journalctl@ceph.osd.7.smithi144.stdout:Sep 16 08:29:06 smithi144 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-7[59146]: 2024-09-16T08:29:06.266+0000 7f4a9aa4d640 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-09-16T08:28:37.859409+0000 front 2024-09-16T08:28:37.859423+0000 (oldest deadline 2024-09-16T08:29:03.759284+0000) 2024-09-16T08:29:06.959 INFO:journalctl@ceph.osd.5.smithi144.stdout:Sep 16 08:29:06 smithi144 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-5[55067]: 2024-09-16T08:29:06.898+0000 7fdc67dd8640 -1 osd.5 158 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-09-16T08:28:37.925923+0000 front 2024-09-16T08:28:37.925909+0000 (oldest deadline 2024-09-16T08:29:02.625815+0000) 2024-09-16T08:29:06.960 INFO:journalctl@ceph.osd.6.smithi144.stdout:Sep 16 08:29:06 smithi144 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-6[57105]: 2024-09-16T08:29:06.661+0000 7fc05286c640 -1 osd.6 158 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-09-16T08:28:38.187159+0000 front 2024-09-16T08:28:38.187196+0000 (oldest deadline 2024-09-16T08:29:02.887071+0000) 2024-09-16T08:29:07.296 INFO:journalctl@ceph.osd.4.smithi144.stdout:Sep 16 08:29:06 smithi144 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-4[53035]: 2024-09-16T08:29:06.959+0000 7f3052327640 -1 osd.4 158 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-09-16T08:28:37.576788+0000 front 2024-09-16T08:28:37.576964+0000 (oldest deadline 2024-09-16T08:29:03.476816+0000) 2024-09-16T08:29:07.582 INFO:journalctl@ceph.osd.7.smithi144.stdout:Sep 16 08:29:07 smithi144 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-7[59146]: 2024-09-16T08:29:07.296+0000 7f4a9aa4d640 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-09-16T08:28:37.859409+0000 front 2024-09-16T08:28:37.859423+0000 (oldest deadline 2024-09-16T08:29:03.759284+0000) 2024-09-16T08:29:07.998 INFO:journalctl@ceph.osd.6.smithi144.stdout:Sep 16 08:29:07 smithi144 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-6[57105]: 2024-09-16T08:29:07.670+0000 7fc05286c640 -1 osd.6 158 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-09-16T08:28:38.187159+0000 front 2024-09-16T08:28:38.187196+0000 (oldest deadline 2024-09-16T08:29:02.887071+0000) 2024-09-16T08:29:07.998 INFO:journalctl@ceph.osd.5.smithi144.stdout:Sep 16 08:29:07 smithi144 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-5[55067]: 2024-09-16T08:29:07.852+0000 7fdc67dd8640 -1 osd.5 158 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-09-16T08:28:37.925923+0000 front 2024-09-16T08:28:37.925909+0000 (oldest deadline 2024-09-16T08:29:02.625815+0000) 2024-09-16T08:29:07.998 INFO:journalctl@ceph.osd.5.smithi144.stdout:Sep 16 08:29:07 smithi144 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-5[55067]: 2024-09-16T08:29:07.852+0000 7fdc67dd8640 -1 osd.5 158 heartbeat_check: no reply from 172.21.15.28:6814 osd.1 since back 2024-09-16T08:28:43.726943+0000 front 2024-09-16T08:28:43.727046+0000 (oldest deadline 2024-09-16T08:29:07.826808+0000) 2024-09-16T08:29:08.294 INFO:journalctl@ceph.osd.4.smithi144.stdout:Sep 16 08:29:07 smithi144 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-4[53035]: 2024-09-16T08:29:07.998+0000 7f3052327640 -1 osd.4 158 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-09-16T08:28:37.576788+0000 front 2024-09-16T08:28:37.576964+0000 (oldest deadline 2024-09-16T08:29:03.476816+0000) 2024-09-16T08:29:08.582 INFO:journalctl@ceph.osd.7.smithi144.stdout:Sep 16 08:29:08 smithi144 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-7[59146]: 2024-09-16T08:29:08.294+0000 7f4a9aa4d640 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-09-16T08:28:37.859409+0000 front 2024-09-16T08:28:37.859423+0000 (oldest deadline 2024-09-16T08:29:03.759284+0000) 2024-09-16T08:29:09.033 INFO:journalctl@ceph.osd.5.smithi144.stdout:Sep 16 08:29:08 smithi144 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-5[55067]: 2024-09-16T08:29:08.887+0000 7fdc67dd8640 -1 osd.5 158 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-09-16T08:28:37.925923+0000 front 2024-09-16T08:28:37.925909+0000 (oldest deadline 2024-09-16T08:29:02.625815+0000) 2024-09-16T08:29:09.033 INFO:journalctl@ceph.osd.5.smithi144.stdout:Sep 16 08:29:08 smithi144 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-5[55067]: 2024-09-16T08:29:08.887+0000 7fdc67dd8640 -1 osd.5 158 heartbeat_check: no reply from 172.21.15.28:6814 osd.1 since back 2024-09-16T08:28:43.726943+0000 front 2024-09-16T08:28:43.727046+0000 (oldest deadline 2024-09-16T08:29:07.826808+0000) 2024-09-16T08:29:09.033 INFO:journalctl@ceph.osd.6.smithi144.stdout:Sep 16 08:29:08 smithi144 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-6[57105]: 2024-09-16T08:29:08.667+0000 7fc05286c640 -1 osd.6 158 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-09-16T08:28:38.187159+0000 front 2024-09-16T08:28:38.187196+0000 (oldest deadline 2024-09-16T08:29:02.887071+0000) 2024-09-16T08:29:09.034 INFO:journalctl@ceph.osd.6.smithi144.stdout:Sep 16 08:29:08 smithi144 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-6[57105]: 2024-09-16T08:29:08.667+0000 7fc05286c640 -1 osd.6 158 heartbeat_check: no reply from 172.21.15.28:6814 osd.1 since back 2024-09-16T08:28:46.789028+0000 front 2024-09-16T08:28:46.789038+0000 (oldest deadline 2024-09-16T08:29:07.888859+0000) 2024-09-16T08:29:09.332 INFO:journalctl@ceph.osd.4.smithi144.stdout:Sep 16 08:29:09 smithi144 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-4[53035]: 2024-09-16T08:29:09.033+0000 7f3052327640 -1 osd.4 158 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-09-16T08:28:37.576788+0000 front 2024-09-16T08:28:37.576964+0000 (oldest deadline 2024-09-16T08:29:03.476816+0000) 2024-09-16T08:29:09.332 INFO:journalctl@ceph.osd.4.smithi144.stdout:Sep 16 08:29:09 smithi144 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-4[53035]: 2024-09-16T08:29:09.033+0000 7f3052327640 -1 osd.4 158 heartbeat_check: no reply from 172.21.15.28:6814 osd.1 since back 2024-09-16T08:28:43.477346+0000 front 2024-09-16T08:28:43.477436+0000 (oldest deadline 2024-09-16T08:29:08.777265+0000) 2024-09-16T08:29:09.332 INFO:journalctl@ceph.osd.7.smithi144.stdout:Sep 16 08:29:09 smithi144 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-7[59146]: 2024-09-16T08:29:09.249+0000 7f4a9aa4d640 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-09-16T08:28:37.859409+0000 front 2024-09-16T08:28:37.859423+0000 (oldest deadline 2024-09-16T08:29:03.759284+0000) 2024-09-16T08:29:09.332 INFO:journalctl@ceph.osd.7.smithi144.stdout:Sep 16 08:29:09 smithi144 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-7[59146]: 2024-09-16T08:29:09.249+0000 7f4a9aa4d640 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.28:6814 osd.1 since back 2024-09-16T08:28:43.760028+0000 front 2024-09-16T08:28:43.759835+0000 (oldest deadline 2024-09-16T08:29:08.459749+0000) 2024-09-16T08:29:10.010 INFO:journalctl@ceph.osd.5.smithi144.stdout:Sep 16 08:29:09 smithi144 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-5[55067]: 2024-09-16T08:29:09.884+0000 7fdc67dd8640 -1 osd.5 158 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-09-16T08:28:37.925923+0000 front 2024-09-16T08:28:37.925909+0000 (oldest deadline 2024-09-16T08:29:02.625815+0000) 2024-09-16T08:29:10.010 INFO:journalctl@ceph.osd.5.smithi144.stdout:Sep 16 08:29:09 smithi144 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-5[55067]: 2024-09-16T08:29:09.884+0000 7fdc67dd8640 -1 osd.5 158 heartbeat_check: no reply from 172.21.15.28:6814 osd.1 since back 2024-09-16T08:28:43.726943+0000 front 2024-09-16T08:28:43.727046+0000 (oldest deadline 2024-09-16T08:29:07.826808+0000) 2024-09-16T08:29:10.010 INFO:journalctl@ceph.osd.6.smithi144.stdout:Sep 16 08:29:09 smithi144 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-6[57105]: 2024-09-16T08:29:09.654+0000 7fc05286c640 -1 osd.6 158 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-09-16T08:28:38.187159+0000 front 2024-09-16T08:28:38.187196+0000 (oldest deadline 2024-09-16T08:29:02.887071+0000) 2024-09-16T08:29:10.010 INFO:journalctl@ceph.osd.6.smithi144.stdout:Sep 16 08:29:09 smithi144 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-6[57105]: 2024-09-16T08:29:09.654+0000 7fc05286c640 -1 osd.6 158 heartbeat_check: no reply from 172.21.15.28:6814 osd.1 since back 2024-09-16T08:28:46.789028+0000 front 2024-09-16T08:28:46.789038+0000 (oldest deadline 2024-09-16T08:29:07.888859+0000) 2024-09-16T08:29:10.290 INFO:journalctl@ceph.osd.4.smithi144.stdout:Sep 16 08:29:10 smithi144 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-4[53035]: 2024-09-16T08:29:10.009+0000 7f3052327640 -1 osd.4 158 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-09-16T08:28:37.576788+0000 front 2024-09-16T08:28:37.576964+0000 (oldest deadline 2024-09-16T08:29:03.476816+0000) 2024-09-16T08:29:10.291 INFO:journalctl@ceph.osd.4.smithi144.stdout:Sep 16 08:29:10 smithi144 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-4[53035]: 2024-09-16T08:29:10.009+0000 7f3052327640 -1 osd.4 158 heartbeat_check: no reply from 172.21.15.28:6814 osd.1 since back 2024-09-16T08:28:43.477346+0000 front 2024-09-16T08:28:43.477436+0000 (oldest deadline 2024-09-16T08:29:08.777265+0000) 2024-09-16T08:29:10.582 INFO:journalctl@ceph.osd.7.smithi144.stdout:Sep 16 08:29:10 smithi144 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-7[59146]: 2024-09-16T08:29:10.289+0000 7f4a9aa4d640 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-09-16T08:28:37.859409+0000 front 2024-09-16T08:28:37.859423+0000 (oldest deadline 2024-09-16T08:29:03.759284+0000) 2024-09-16T08:29:10.582 INFO:journalctl@ceph.osd.7.smithi144.stdout:Sep 16 08:29:10 smithi144 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-7[59146]: 2024-09-16T08:29:10.289+0000 7f4a9aa4d640 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.28:6814 osd.1 since back 2024-09-16T08:28:43.760028+0000 front 2024-09-16T08:28:43.759835+0000 (oldest deadline 2024-09-16T08:29:08.459749+0000) 2024-09-16T08:29:11.082 INFO:journalctl@ceph.osd.4.smithi144.stdout:Sep 16 08:29:10 smithi144 podman[68710]: 2024-09-16 08:29:10.810177528 +0000 UTC m=+5.513657648 container died c48cb68e901e50e238853991ed735fc87807751260f575193166aead0e2ac74f (image=quay.ceph.io/ceph-ci/ceph@sha256:f7ad55f9ccd17ba66aadb19b4d291e67b60ea24e5d8a0c0e92941fd59c81e4ce, name=ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-4, org.label-schema.vendor=CentOS, ceph=True, CEPH_POINT_RELEASE=, RELEASE=wip-guits-main-2024-09-13-1248-8293d73, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965) 2024-09-16T08:29:11.083 INFO:journalctl@ceph.osd.5.smithi144.stdout:Sep 16 08:29:10 smithi144 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-5[55067]: 2024-09-16T08:29:10.838+0000 7fdc67dd8640 -1 osd.5 158 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-09-16T08:28:37.925923+0000 front 2024-09-16T08:28:37.925909+0000 (oldest deadline 2024-09-16T08:29:02.625815+0000) 2024-09-16T08:29:11.083 INFO:journalctl@ceph.osd.5.smithi144.stdout:Sep 16 08:29:10 smithi144 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-5[55067]: 2024-09-16T08:29:10.838+0000 7fdc67dd8640 -1 osd.5 158 heartbeat_check: no reply from 172.21.15.28:6814 osd.1 since back 2024-09-16T08:28:43.726943+0000 front 2024-09-16T08:28:43.727046+0000 (oldest deadline 2024-09-16T08:29:07.826808+0000) 2024-09-16T08:29:11.084 INFO:journalctl@ceph.osd.6.smithi144.stdout:Sep 16 08:29:10 smithi144 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-6[57105]: 2024-09-16T08:29:10.610+0000 7fc05286c640 -1 osd.6 158 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-09-16T08:28:38.187159+0000 front 2024-09-16T08:28:38.187196+0000 (oldest deadline 2024-09-16T08:29:02.887071+0000) 2024-09-16T08:29:11.084 INFO:journalctl@ceph.osd.6.smithi144.stdout:Sep 16 08:29:10 smithi144 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-6[57105]: 2024-09-16T08:29:10.610+0000 7fc05286c640 -1 osd.6 158 heartbeat_check: no reply from 172.21.15.28:6814 osd.1 since back 2024-09-16T08:28:46.789028+0000 front 2024-09-16T08:28:46.789038+0000 (oldest deadline 2024-09-16T08:29:07.888859+0000) 2024-09-16T08:29:11.555 INFO:journalctl@ceph.osd.4.smithi144.stdout:Sep 16 08:29:11 smithi144 podman[68710]: 2024-09-16 08:29:11.295866459 +0000 UTC m=+5.999346559 container cleanup c48cb68e901e50e238853991ed735fc87807751260f575193166aead0e2ac74f (image=quay.ceph.io/ceph-ci/ceph@sha256:f7ad55f9ccd17ba66aadb19b4d291e67b60ea24e5d8a0c0e92941fd59c81e4ce, name=ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-4, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_CLEAN=True, RELEASE=wip-guits-main-2024-09-13-1248-8293d73, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=) 2024-09-16T08:29:11.555 INFO:journalctl@ceph.osd.4.smithi144.stdout:Sep 16 08:29:11 smithi144 bash[68710]: ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-4 2024-09-16T08:29:11.556 INFO:journalctl@ceph.osd.7.smithi144.stdout:Sep 16 08:29:11 smithi144 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-7[59146]: 2024-09-16T08:29:11.328+0000 7f4a9aa4d640 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-09-16T08:28:37.859409+0000 front 2024-09-16T08:28:37.859423+0000 (oldest deadline 2024-09-16T08:29:03.759284+0000) 2024-09-16T08:29:11.556 INFO:journalctl@ceph.osd.7.smithi144.stdout:Sep 16 08:29:11 smithi144 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-7[59146]: 2024-09-16T08:29:11.328+0000 7f4a9aa4d640 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.28:6814 osd.1 since back 2024-09-16T08:28:43.760028+0000 front 2024-09-16T08:28:43.759835+0000 (oldest deadline 2024-09-16T08:29:08.459749+0000) 2024-09-16T08:29:11.832 INFO:journalctl@ceph.osd.4.smithi144.stdout:Sep 16 08:29:11 smithi144 podman[68726]: 2024-09-16 08:29:11.555756306 +0000 UTC m=+0.740684660 container remove c48cb68e901e50e238853991ed735fc87807751260f575193166aead0e2ac74f (image=quay.ceph.io/ceph-ci/ceph@sha256:f7ad55f9ccd17ba66aadb19b4d291e67b60ea24e5d8a0c0e92941fd59c81e4ce, name=ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-4, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.schema-version=1.0, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, RELEASE=wip-guits-main-2024-09-13-1248-8293d73, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, GIT_CLEAN=True) 2024-09-16T08:29:11.832 INFO:journalctl@ceph.osd.6.smithi144.stdout:Sep 16 08:29:11 smithi144 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-6[57105]: 2024-09-16T08:29:11.636+0000 7fc05286c640 -1 osd.6 158 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-09-16T08:28:38.187159+0000 front 2024-09-16T08:28:38.187196+0000 (oldest deadline 2024-09-16T08:29:02.887071+0000) 2024-09-16T08:29:11.832 INFO:journalctl@ceph.osd.6.smithi144.stdout:Sep 16 08:29:11 smithi144 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-6[57105]: 2024-09-16T08:29:11.636+0000 7fc05286c640 -1 osd.6 158 heartbeat_check: no reply from 172.21.15.28:6814 osd.1 since back 2024-09-16T08:28:46.789028+0000 front 2024-09-16T08:28:46.789038+0000 (oldest deadline 2024-09-16T08:29:07.888859+0000) 2024-09-16T08:29:12.184 INFO:journalctl@ceph.osd.5.smithi144.stdout:Sep 16 08:29:11 smithi144 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-5[55067]: 2024-09-16T08:29:11.855+0000 7fdc67dd8640 -1 osd.5 158 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-09-16T08:28:37.925923+0000 front 2024-09-16T08:28:37.925909+0000 (oldest deadline 2024-09-16T08:29:02.625815+0000) 2024-09-16T08:29:12.184 INFO:journalctl@ceph.osd.5.smithi144.stdout:Sep 16 08:29:11 smithi144 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-5[55067]: 2024-09-16T08:29:11.855+0000 7fdc67dd8640 -1 osd.5 158 heartbeat_check: no reply from 172.21.15.28:6814 osd.1 since back 2024-09-16T08:28:43.726943+0000 front 2024-09-16T08:28:43.727046+0000 (oldest deadline 2024-09-16T08:29:07.826808+0000) 2024-09-16T08:29:12.184 INFO:journalctl@ceph.osd.4.smithi144.stdout:Sep 16 08:29:11 smithi144 podman[68817]: 2024-09-16 08:29:11.8136016 +0000 UTC m=+0.015850913 image pull 425e316ba8ebf74f04aa3640f7deb4fa3f293a78d0c8d097cb679eaf86ddca6c quay.ceph.io/ceph-ci/ceph@sha256:f7ad55f9ccd17ba66aadb19b4d291e67b60ea24e5d8a0c0e92941fd59c81e4ce 2024-09-16T08:29:12.184 INFO:journalctl@ceph.osd.4.smithi144.stdout:Sep 16 08:29:11 smithi144 podman[68817]: 2024-09-16 08:29:11.940180167 +0000 UTC m=+0.142429463 container create 73d163131f51f59a915ace3018864ff2f36e92a1abf22a017f6f4d52429a4716 (image=quay.ceph.io/ceph-ci/ceph@sha256:f7ad55f9ccd17ba66aadb19b4d291e67b60ea24e5d8a0c0e92941fd59c81e4ce, name=ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-4-deactivate, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=wip-guits-main-2024-09-13-1248-8293d73, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_CLEAN=True, ceph=True, maintainer=Guillaume Abrioux ) 2024-09-16T08:29:12.185 INFO:journalctl@ceph.osd.4.smithi144.stdout:Sep 16 08:29:12 smithi144 podman[68817]: 2024-09-16 08:29:12.184852227 +0000 UTC m=+0.387101528 container init 73d163131f51f59a915ace3018864ff2f36e92a1abf22a017f6f4d52429a4716 (image=quay.ceph.io/ceph-ci/ceph@sha256:f7ad55f9ccd17ba66aadb19b4d291e67b60ea24e5d8a0c0e92941fd59c81e4ce, name=ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-4-deactivate, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , RELEASE=wip-guits-main-2024-09-13-1248-8293d73, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0) 2024-09-16T08:29:12.582 INFO:journalctl@ceph.osd.7.smithi144.stdout:Sep 16 08:29:12 smithi144 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-7[59146]: 2024-09-16T08:29:12.306+0000 7f4a9aa4d640 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-09-16T08:28:37.859409+0000 front 2024-09-16T08:28:37.859423+0000 (oldest deadline 2024-09-16T08:29:03.759284+0000) 2024-09-16T08:29:12.582 INFO:journalctl@ceph.osd.7.smithi144.stdout:Sep 16 08:29:12 smithi144 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-7[59146]: 2024-09-16T08:29:12.306+0000 7f4a9aa4d640 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.28:6814 osd.1 since back 2024-09-16T08:28:43.760028+0000 front 2024-09-16T08:28:43.759835+0000 (oldest deadline 2024-09-16T08:29:08.459749+0000) 2024-09-16T08:29:12.582 INFO:journalctl@ceph.osd.4.smithi144.stdout:Sep 16 08:29:12 smithi144 podman[68817]: 2024-09-16 08:29:12.1885185 +0000 UTC m=+0.390767798 container start 73d163131f51f59a915ace3018864ff2f36e92a1abf22a017f6f4d52429a4716 (image=quay.ceph.io/ceph-ci/ceph@sha256:f7ad55f9ccd17ba66aadb19b4d291e67b60ea24e5d8a0c0e92941fd59c81e4ce, name=ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-4-deactivate, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, RELEASE=wip-guits-main-2024-09-13-1248-8293d73, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_CLEAN=True, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, ceph=True, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=) 2024-09-16T08:29:12.582 INFO:journalctl@ceph.osd.4.smithi144.stdout:Sep 16 08:29:12 smithi144 podman[68817]: 2024-09-16 08:29:12.259931392 +0000 UTC m=+0.462180698 container attach 73d163131f51f59a915ace3018864ff2f36e92a1abf22a017f6f4d52429a4716 (image=quay.ceph.io/ceph-ci/ceph@sha256:f7ad55f9ccd17ba66aadb19b4d291e67b60ea24e5d8a0c0e92941fd59c81e4ce, name=ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-4-deactivate, org.label-schema.license=GPLv2, GIT_CLEAN=True, RELEASE=wip-guits-main-2024-09-13-1248-8293d73, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS) 2024-09-16T08:29:12.582 INFO:journalctl@ceph.osd.4.smithi144.stdout:Sep 16 08:29:12 smithi144 podman[68817]: 2024-09-16 08:29:12.395555007 +0000 UTC m=+0.597804315 container died 73d163131f51f59a915ace3018864ff2f36e92a1abf22a017f6f4d52429a4716 (image=quay.ceph.io/ceph-ci/ceph@sha256:f7ad55f9ccd17ba66aadb19b4d291e67b60ea24e5d8a0c0e92941fd59c81e4ce, name=ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-4-deactivate, GIT_CLEAN=True, RELEASE=wip-guits-main-2024-09-13-1248-8293d73, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, ceph=True, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965) 2024-09-16T08:29:12.891 INFO:journalctl@ceph.osd.6.smithi144.stdout:Sep 16 08:29:12 smithi144 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-6[57105]: 2024-09-16T08:29:12.609+0000 7fc05286c640 -1 osd.6 158 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-09-16T08:28:38.187159+0000 front 2024-09-16T08:28:38.187196+0000 (oldest deadline 2024-09-16T08:29:02.887071+0000) 2024-09-16T08:29:12.891 INFO:journalctl@ceph.osd.6.smithi144.stdout:Sep 16 08:29:12 smithi144 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-6[57105]: 2024-09-16T08:29:12.609+0000 7fc05286c640 -1 osd.6 158 heartbeat_check: no reply from 172.21.15.28:6814 osd.1 since back 2024-09-16T08:28:46.789028+0000 front 2024-09-16T08:28:46.789038+0000 (oldest deadline 2024-09-16T08:29:07.888859+0000) 2024-09-16T08:29:13.014 DEBUG:teuthology.orchestra.run.smithi144:> sudo pkill -f 'journalctl -f -n 0 -u ceph-5622b2fe-7402-11ef-bceb-c7b262605968@osd.4.service' 2024-09-16T08:29:13.053 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-16T08:29:13.053 INFO:tasks.cephadm.osd.4:Stopped osd.4 2024-09-16T08:29:13.053 INFO:tasks.cephadm.osd.5:Stopping osd.5... 2024-09-16T08:29:13.053 DEBUG:teuthology.orchestra.run.smithi144:> sudo systemctl stop ceph-5622b2fe-7402-11ef-bceb-c7b262605968@osd.5 2024-09-16T08:29:13.313 INFO:journalctl@ceph.osd.5.smithi144.stdout:Sep 16 08:29:12 smithi144 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-5[55067]: 2024-09-16T08:29:12.890+0000 7fdc67dd8640 -1 osd.5 158 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-09-16T08:28:37.925923+0000 front 2024-09-16T08:28:37.925909+0000 (oldest deadline 2024-09-16T08:29:02.625815+0000) 2024-09-16T08:29:13.313 INFO:journalctl@ceph.osd.5.smithi144.stdout:Sep 16 08:29:12 smithi144 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-5[55067]: 2024-09-16T08:29:12.890+0000 7fdc67dd8640 -1 osd.5 158 heartbeat_check: no reply from 172.21.15.28:6814 osd.1 since back 2024-09-16T08:28:43.726943+0000 front 2024-09-16T08:28:43.727046+0000 (oldest deadline 2024-09-16T08:29:07.826808+0000) 2024-09-16T08:29:13.313 INFO:journalctl@ceph.osd.5.smithi144.stdout:Sep 16 08:29:13 smithi144 systemd[1]: Stopping Ceph osd.5 for 5622b2fe-7402-11ef-bceb-c7b262605968... 2024-09-16T08:29:13.582 INFO:journalctl@ceph.osd.7.smithi144.stdout:Sep 16 08:29:13 smithi144 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-7[59146]: 2024-09-16T08:29:13.310+0000 7f4a9aa4d640 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-09-16T08:28:37.859409+0000 front 2024-09-16T08:28:37.859423+0000 (oldest deadline 2024-09-16T08:29:03.759284+0000) 2024-09-16T08:29:13.582 INFO:journalctl@ceph.osd.7.smithi144.stdout:Sep 16 08:29:13 smithi144 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-7[59146]: 2024-09-16T08:29:13.310+0000 7f4a9aa4d640 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.28:6814 osd.1 since back 2024-09-16T08:28:43.760028+0000 front 2024-09-16T08:28:43.759835+0000 (oldest deadline 2024-09-16T08:29:08.459749+0000) 2024-09-16T08:29:13.582 INFO:journalctl@ceph.osd.5.smithi144.stdout:Sep 16 08:29:13 smithi144 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-5[55067]: 2024-09-16T08:29:13.416+0000 7fdc6b7c3640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.5 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-16T08:29:13.582 INFO:journalctl@ceph.osd.5.smithi144.stdout:Sep 16 08:29:13 smithi144 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-5[55067]: 2024-09-16T08:29:13.416+0000 7fdc6b7c3640 -1 osd.5 158 *** Got signal Terminated *** 2024-09-16T08:29:13.582 INFO:journalctl@ceph.osd.5.smithi144.stdout:Sep 16 08:29:13 smithi144 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-5[55067]: 2024-09-16T08:29:13.416+0000 7fdc6b7c3640 -1 osd.5 158 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-16T08:29:13.849 INFO:journalctl@ceph.osd.6.smithi144.stdout:Sep 16 08:29:13 smithi144 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-6[57105]: 2024-09-16T08:29:13.591+0000 7fc05286c640 -1 osd.6 158 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-09-16T08:28:38.187159+0000 front 2024-09-16T08:28:38.187196+0000 (oldest deadline 2024-09-16T08:29:02.887071+0000) 2024-09-16T08:29:13.849 INFO:journalctl@ceph.osd.6.smithi144.stdout:Sep 16 08:29:13 smithi144 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-6[57105]: 2024-09-16T08:29:13.591+0000 7fc05286c640 -1 osd.6 158 heartbeat_check: no reply from 172.21.15.28:6814 osd.1 since back 2024-09-16T08:28:46.789028+0000 front 2024-09-16T08:28:46.789038+0000 (oldest deadline 2024-09-16T08:29:07.888859+0000) 2024-09-16T08:29:14.280 INFO:journalctl@ceph.osd.5.smithi144.stdout:Sep 16 08:29:13 smithi144 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-5[55067]: 2024-09-16T08:29:13.848+0000 7fdc67dd8640 -1 osd.5 158 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-09-16T08:28:37.925923+0000 front 2024-09-16T08:28:37.925909+0000 (oldest deadline 2024-09-16T08:29:02.625815+0000) 2024-09-16T08:29:14.280 INFO:journalctl@ceph.osd.5.smithi144.stdout:Sep 16 08:29:13 smithi144 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-5[55067]: 2024-09-16T08:29:13.848+0000 7fdc67dd8640 -1 osd.5 158 heartbeat_check: no reply from 172.21.15.28:6814 osd.1 since back 2024-09-16T08:28:43.726943+0000 front 2024-09-16T08:28:43.727046+0000 (oldest deadline 2024-09-16T08:29:07.826808+0000) 2024-09-16T08:29:14.582 INFO:journalctl@ceph.osd.7.smithi144.stdout:Sep 16 08:29:14 smithi144 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-7[59146]: 2024-09-16T08:29:14.280+0000 7f4a9aa4d640 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-09-16T08:28:37.859409+0000 front 2024-09-16T08:28:37.859423+0000 (oldest deadline 2024-09-16T08:29:03.759284+0000) 2024-09-16T08:29:14.582 INFO:journalctl@ceph.osd.7.smithi144.stdout:Sep 16 08:29:14 smithi144 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-7[59146]: 2024-09-16T08:29:14.280+0000 7f4a9aa4d640 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.28:6814 osd.1 since back 2024-09-16T08:28:43.760028+0000 front 2024-09-16T08:28:43.759835+0000 (oldest deadline 2024-09-16T08:29:08.459749+0000) 2024-09-16T08:29:15.082 INFO:journalctl@ceph.osd.5.smithi144.stdout:Sep 16 08:29:14 smithi144 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-5[55067]: 2024-09-16T08:29:14.839+0000 7fdc67dd8640 -1 osd.5 158 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-09-16T08:28:37.925923+0000 front 2024-09-16T08:28:37.925909+0000 (oldest deadline 2024-09-16T08:29:02.625815+0000) 2024-09-16T08:29:15.082 INFO:journalctl@ceph.osd.5.smithi144.stdout:Sep 16 08:29:14 smithi144 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-5[55067]: 2024-09-16T08:29:14.839+0000 7fdc67dd8640 -1 osd.5 158 heartbeat_check: no reply from 172.21.15.28:6814 osd.1 since back 2024-09-16T08:28:43.726943+0000 front 2024-09-16T08:28:43.727046+0000 (oldest deadline 2024-09-16T08:29:07.826808+0000) 2024-09-16T08:29:15.083 INFO:journalctl@ceph.osd.6.smithi144.stdout:Sep 16 08:29:14 smithi144 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-6[57105]: 2024-09-16T08:29:14.604+0000 7fc05286c640 -1 osd.6 158 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-09-16T08:28:38.187159+0000 front 2024-09-16T08:28:38.187196+0000 (oldest deadline 2024-09-16T08:29:02.887071+0000) 2024-09-16T08:29:15.083 INFO:journalctl@ceph.osd.6.smithi144.stdout:Sep 16 08:29:14 smithi144 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-6[57105]: 2024-09-16T08:29:14.604+0000 7fc05286c640 -1 osd.6 158 heartbeat_check: no reply from 172.21.15.28:6814 osd.1 since back 2024-09-16T08:28:46.789028+0000 front 2024-09-16T08:28:46.789038+0000 (oldest deadline 2024-09-16T08:29:07.888859+0000) 2024-09-16T08:29:15.582 INFO:journalctl@ceph.osd.7.smithi144.stdout:Sep 16 08:29:15 smithi144 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-7[59146]: 2024-09-16T08:29:15.267+0000 7f4a9aa4d640 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-09-16T08:28:37.859409+0000 front 2024-09-16T08:28:37.859423+0000 (oldest deadline 2024-09-16T08:29:03.759284+0000) 2024-09-16T08:29:15.582 INFO:journalctl@ceph.osd.7.smithi144.stdout:Sep 16 08:29:15 smithi144 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-7[59146]: 2024-09-16T08:29:15.267+0000 7f4a9aa4d640 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.28:6814 osd.1 since back 2024-09-16T08:28:43.760028+0000 front 2024-09-16T08:28:43.759835+0000 (oldest deadline 2024-09-16T08:29:08.459749+0000) 2024-09-16T08:29:16.081 INFO:journalctl@ceph.osd.5.smithi144.stdout:Sep 16 08:29:15 smithi144 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-5[55067]: 2024-09-16T08:29:15.888+0000 7fdc67dd8640 -1 osd.5 158 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-09-16T08:28:37.925923+0000 front 2024-09-16T08:28:37.925909+0000 (oldest deadline 2024-09-16T08:29:02.625815+0000) 2024-09-16T08:29:16.082 INFO:journalctl@ceph.osd.5.smithi144.stdout:Sep 16 08:29:15 smithi144 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-5[55067]: 2024-09-16T08:29:15.888+0000 7fdc67dd8640 -1 osd.5 158 heartbeat_check: no reply from 172.21.15.28:6814 osd.1 since back 2024-09-16T08:28:43.726943+0000 front 2024-09-16T08:28:43.727046+0000 (oldest deadline 2024-09-16T08:29:07.826808+0000) 2024-09-16T08:29:16.082 INFO:journalctl@ceph.osd.6.smithi144.stdout:Sep 16 08:29:15 smithi144 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-6[57105]: 2024-09-16T08:29:15.652+0000 7fc05286c640 -1 osd.6 158 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-09-16T08:28:38.187159+0000 front 2024-09-16T08:28:38.187196+0000 (oldest deadline 2024-09-16T08:29:02.887071+0000) 2024-09-16T08:29:16.082 INFO:journalctl@ceph.osd.6.smithi144.stdout:Sep 16 08:29:15 smithi144 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-6[57105]: 2024-09-16T08:29:15.652+0000 7fc05286c640 -1 osd.6 158 heartbeat_check: no reply from 172.21.15.28:6814 osd.1 since back 2024-09-16T08:28:46.789028+0000 front 2024-09-16T08:28:46.789038+0000 (oldest deadline 2024-09-16T08:29:07.888859+0000) 2024-09-16T08:29:16.582 INFO:journalctl@ceph.osd.7.smithi144.stdout:Sep 16 08:29:16 smithi144 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-7[59146]: 2024-09-16T08:29:16.243+0000 7f4a9aa4d640 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-09-16T08:28:37.859409+0000 front 2024-09-16T08:28:37.859423+0000 (oldest deadline 2024-09-16T08:29:03.759284+0000) 2024-09-16T08:29:16.582 INFO:journalctl@ceph.osd.7.smithi144.stdout:Sep 16 08:29:16 smithi144 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-7[59146]: 2024-09-16T08:29:16.243+0000 7f4a9aa4d640 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.28:6814 osd.1 since back 2024-09-16T08:28:43.760028+0000 front 2024-09-16T08:28:43.759835+0000 (oldest deadline 2024-09-16T08:29:08.459749+0000) 2024-09-16T08:29:16.582 INFO:journalctl@ceph.osd.7.smithi144.stdout:Sep 16 08:29:16 smithi144 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-7[59146]: 2024-09-16T08:29:16.243+0000 7f4a9aa4d640 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.28:6822 osd.2 since back 2024-09-16T08:28:53.160694+0000 front 2024-09-16T08:28:53.160755+0000 (oldest deadline 2024-09-16T08:29:15.460707+0000) 2024-09-16T08:29:16.877 INFO:journalctl@ceph.osd.6.smithi144.stdout:Sep 16 08:29:16 smithi144 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-6[57105]: 2024-09-16T08:29:16.614+0000 7fc05286c640 -1 osd.6 158 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-09-16T08:28:38.187159+0000 front 2024-09-16T08:28:38.187196+0000 (oldest deadline 2024-09-16T08:29:02.887071+0000) 2024-09-16T08:29:16.877 INFO:journalctl@ceph.osd.6.smithi144.stdout:Sep 16 08:29:16 smithi144 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-6[57105]: 2024-09-16T08:29:16.614+0000 7fc05286c640 -1 osd.6 158 heartbeat_check: no reply from 172.21.15.28:6814 osd.1 since back 2024-09-16T08:28:46.789028+0000 front 2024-09-16T08:28:46.789038+0000 (oldest deadline 2024-09-16T08:29:07.888859+0000) 2024-09-16T08:29:17.206 INFO:journalctl@ceph.osd.5.smithi144.stdout:Sep 16 08:29:16 smithi144 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-5[55067]: 2024-09-16T08:29:16.876+0000 7fdc67dd8640 -1 osd.5 158 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-09-16T08:28:37.925923+0000 front 2024-09-16T08:28:37.925909+0000 (oldest deadline 2024-09-16T08:29:02.625815+0000) 2024-09-16T08:29:17.206 INFO:journalctl@ceph.osd.5.smithi144.stdout:Sep 16 08:29:16 smithi144 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-5[55067]: 2024-09-16T08:29:16.876+0000 7fdc67dd8640 -1 osd.5 158 heartbeat_check: no reply from 172.21.15.28:6814 osd.1 since back 2024-09-16T08:28:43.726943+0000 front 2024-09-16T08:28:43.727046+0000 (oldest deadline 2024-09-16T08:29:07.826808+0000) 2024-09-16T08:29:17.206 INFO:journalctl@ceph.osd.5.smithi144.stdout:Sep 16 08:29:16 smithi144 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-5[55067]: 2024-09-16T08:29:16.876+0000 7fdc67dd8640 -1 osd.5 158 heartbeat_check: no reply from 172.21.15.28:6822 osd.2 since back 2024-09-16T08:28:50.727570+0000 front 2024-09-16T08:28:50.727554+0000 (oldest deadline 2024-09-16T08:29:16.027488+0000) 2024-09-16T08:29:17.582 INFO:journalctl@ceph.osd.7.smithi144.stdout:Sep 16 08:29:17 smithi144 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-7[59146]: 2024-09-16T08:29:17.205+0000 7f4a9aa4d640 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-09-16T08:28:37.859409+0000 front 2024-09-16T08:28:37.859423+0000 (oldest deadline 2024-09-16T08:29:03.759284+0000) 2024-09-16T08:29:17.582 INFO:journalctl@ceph.osd.7.smithi144.stdout:Sep 16 08:29:17 smithi144 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-7[59146]: 2024-09-16T08:29:17.205+0000 7f4a9aa4d640 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.28:6814 osd.1 since back 2024-09-16T08:28:43.760028+0000 front 2024-09-16T08:28:43.759835+0000 (oldest deadline 2024-09-16T08:29:08.459749+0000) 2024-09-16T08:29:17.582 INFO:journalctl@ceph.osd.7.smithi144.stdout:Sep 16 08:29:17 smithi144 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-7[59146]: 2024-09-16T08:29:17.205+0000 7f4a9aa4d640 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.28:6822 osd.2 since back 2024-09-16T08:28:53.160694+0000 front 2024-09-16T08:28:53.160755+0000 (oldest deadline 2024-09-16T08:29:15.460707+0000) 2024-09-16T08:29:17.893 INFO:journalctl@ceph.osd.6.smithi144.stdout:Sep 16 08:29:17 smithi144 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-6[57105]: 2024-09-16T08:29:17.628+0000 7fc05286c640 -1 osd.6 158 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-09-16T08:28:38.187159+0000 front 2024-09-16T08:28:38.187196+0000 (oldest deadline 2024-09-16T08:29:02.887071+0000) 2024-09-16T08:29:17.894 INFO:journalctl@ceph.osd.6.smithi144.stdout:Sep 16 08:29:17 smithi144 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-6[57105]: 2024-09-16T08:29:17.628+0000 7fc05286c640 -1 osd.6 158 heartbeat_check: no reply from 172.21.15.28:6814 osd.1 since back 2024-09-16T08:28:46.789028+0000 front 2024-09-16T08:28:46.789038+0000 (oldest deadline 2024-09-16T08:29:07.888859+0000) 2024-09-16T08:29:18.226 INFO:journalctl@ceph.osd.5.smithi144.stdout:Sep 16 08:29:17 smithi144 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-5[55067]: 2024-09-16T08:29:17.892+0000 7fdc67dd8640 -1 osd.5 158 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-09-16T08:28:37.925923+0000 front 2024-09-16T08:28:37.925909+0000 (oldest deadline 2024-09-16T08:29:02.625815+0000) 2024-09-16T08:29:18.226 INFO:journalctl@ceph.osd.5.smithi144.stdout:Sep 16 08:29:17 smithi144 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-5[55067]: 2024-09-16T08:29:17.892+0000 7fdc67dd8640 -1 osd.5 158 heartbeat_check: no reply from 172.21.15.28:6814 osd.1 since back 2024-09-16T08:28:43.726943+0000 front 2024-09-16T08:28:43.727046+0000 (oldest deadline 2024-09-16T08:29:07.826808+0000) 2024-09-16T08:29:18.226 INFO:journalctl@ceph.osd.5.smithi144.stdout:Sep 16 08:29:17 smithi144 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-5[55067]: 2024-09-16T08:29:17.892+0000 7fdc67dd8640 -1 osd.5 158 heartbeat_check: no reply from 172.21.15.28:6822 osd.2 since back 2024-09-16T08:28:50.727570+0000 front 2024-09-16T08:28:50.727554+0000 (oldest deadline 2024-09-16T08:29:16.027488+0000) 2024-09-16T08:29:18.582 INFO:journalctl@ceph.osd.7.smithi144.stdout:Sep 16 08:29:18 smithi144 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-7[59146]: 2024-09-16T08:29:18.225+0000 7f4a9aa4d640 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-09-16T08:28:37.859409+0000 front 2024-09-16T08:28:37.859423+0000 (oldest deadline 2024-09-16T08:29:03.759284+0000) 2024-09-16T08:29:18.582 INFO:journalctl@ceph.osd.7.smithi144.stdout:Sep 16 08:29:18 smithi144 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-7[59146]: 2024-09-16T08:29:18.225+0000 7f4a9aa4d640 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.28:6814 osd.1 since back 2024-09-16T08:28:43.760028+0000 front 2024-09-16T08:28:43.759835+0000 (oldest deadline 2024-09-16T08:29:08.459749+0000) 2024-09-16T08:29:18.582 INFO:journalctl@ceph.osd.7.smithi144.stdout:Sep 16 08:29:18 smithi144 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-7[59146]: 2024-09-16T08:29:18.225+0000 7f4a9aa4d640 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.28:6822 osd.2 since back 2024-09-16T08:28:53.160694+0000 front 2024-09-16T08:28:53.160755+0000 (oldest deadline 2024-09-16T08:29:15.460707+0000) 2024-09-16T08:29:18.969 INFO:journalctl@ceph.osd.6.smithi144.stdout:Sep 16 08:29:18 smithi144 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-6[57105]: 2024-09-16T08:29:18.617+0000 7fc05286c640 -1 osd.6 158 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-09-16T08:28:38.187159+0000 front 2024-09-16T08:28:38.187196+0000 (oldest deadline 2024-09-16T08:29:02.887071+0000) 2024-09-16T08:29:18.969 INFO:journalctl@ceph.osd.6.smithi144.stdout:Sep 16 08:29:18 smithi144 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-6[57105]: 2024-09-16T08:29:18.617+0000 7fc05286c640 -1 osd.6 158 heartbeat_check: no reply from 172.21.15.28:6814 osd.1 since back 2024-09-16T08:28:46.789028+0000 front 2024-09-16T08:28:46.789038+0000 (oldest deadline 2024-09-16T08:29:07.888859+0000) 2024-09-16T08:29:19.283 INFO:journalctl@ceph.osd.5.smithi144.stdout:Sep 16 08:29:18 smithi144 podman[68931]: 2024-09-16 08:29:18.969107304 +0000 UTC m=+5.681464730 container died 59ec97fb3cf3b1e0948a5d9c63d1c367ba6a57bb7ec3c9822ae00c6637705ab7 (image=quay.ceph.io/ceph-ci/ceph@sha256:f7ad55f9ccd17ba66aadb19b4d291e67b60ea24e5d8a0c0e92941fd59c81e4ce, name=ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-5, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, RELEASE=wip-guits-main-2024-09-13-1248-8293d73, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, ceph=True, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, org.label-schema.license=GPLv2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965) 2024-09-16T08:29:19.284 INFO:journalctl@ceph.osd.7.smithi144.stdout:Sep 16 08:29:19 smithi144 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-7[59146]: 2024-09-16T08:29:19.198+0000 7f4a9aa4d640 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-09-16T08:28:37.859409+0000 front 2024-09-16T08:28:37.859423+0000 (oldest deadline 2024-09-16T08:29:03.759284+0000) 2024-09-16T08:29:19.284 INFO:journalctl@ceph.osd.7.smithi144.stdout:Sep 16 08:29:19 smithi144 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-7[59146]: 2024-09-16T08:29:19.198+0000 7f4a9aa4d640 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.28:6814 osd.1 since back 2024-09-16T08:28:43.760028+0000 front 2024-09-16T08:28:43.759835+0000 (oldest deadline 2024-09-16T08:29:08.459749+0000) 2024-09-16T08:29:19.284 INFO:journalctl@ceph.osd.7.smithi144.stdout:Sep 16 08:29:19 smithi144 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-7[59146]: 2024-09-16T08:29:19.198+0000 7f4a9aa4d640 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.28:6822 osd.2 since back 2024-09-16T08:28:53.160694+0000 front 2024-09-16T08:28:53.160755+0000 (oldest deadline 2024-09-16T08:29:15.460707+0000) 2024-09-16T08:29:19.573 INFO:journalctl@ceph.osd.5.smithi144.stdout:Sep 16 08:29:19 smithi144 podman[68931]: 2024-09-16 08:29:19.492101035 +0000 UTC m=+6.204458451 container cleanup 59ec97fb3cf3b1e0948a5d9c63d1c367ba6a57bb7ec3c9822ae00c6637705ab7 (image=quay.ceph.io/ceph-ci/ceph@sha256:f7ad55f9ccd17ba66aadb19b4d291e67b60ea24e5d8a0c0e92941fd59c81e4ce, name=ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-5, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, CEPH_POINT_RELEASE=, RELEASE=wip-guits-main-2024-09-13-1248-8293d73, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_CLEAN=True, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, ceph=True) 2024-09-16T08:29:19.573 INFO:journalctl@ceph.osd.5.smithi144.stdout:Sep 16 08:29:19 smithi144 bash[68931]: ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-5 2024-09-16T08:29:19.832 INFO:journalctl@ceph.osd.5.smithi144.stdout:Sep 16 08:29:19 smithi144 podman[68965]: 2024-09-16 08:29:19.735282606 +0000 UTC m=+0.763992860 container remove 59ec97fb3cf3b1e0948a5d9c63d1c367ba6a57bb7ec3c9822ae00c6637705ab7 (image=quay.ceph.io/ceph-ci/ceph@sha256:f7ad55f9ccd17ba66aadb19b4d291e67b60ea24e5d8a0c0e92941fd59c81e4ce, name=ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-5, GIT_CLEAN=True, RELEASE=wip-guits-main-2024-09-13-1248-8293d73, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965) 2024-09-16T08:29:19.832 INFO:journalctl@ceph.osd.6.smithi144.stdout:Sep 16 08:29:19 smithi144 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-6[57105]: 2024-09-16T08:29:19.572+0000 7fc05286c640 -1 osd.6 158 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-09-16T08:28:38.187159+0000 front 2024-09-16T08:28:38.187196+0000 (oldest deadline 2024-09-16T08:29:02.887071+0000) 2024-09-16T08:29:19.832 INFO:journalctl@ceph.osd.6.smithi144.stdout:Sep 16 08:29:19 smithi144 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-6[57105]: 2024-09-16T08:29:19.572+0000 7fc05286c640 -1 osd.6 158 heartbeat_check: no reply from 172.21.15.28:6814 osd.1 since back 2024-09-16T08:28:46.789028+0000 front 2024-09-16T08:28:46.789038+0000 (oldest deadline 2024-09-16T08:29:07.888859+0000) 2024-09-16T08:29:19.832 INFO:journalctl@ceph.osd.6.smithi144.stdout:Sep 16 08:29:19 smithi144 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-6[57105]: 2024-09-16T08:29:19.572+0000 7fc05286c640 -1 osd.6 158 heartbeat_check: no reply from 172.21.15.28:6822 osd.2 since back 2024-09-16T08:28:53.590485+0000 front 2024-09-16T08:28:53.590400+0000 (oldest deadline 2024-09-16T08:29:19.490381+0000) 2024-09-16T08:29:20.446 INFO:journalctl@ceph.osd.7.smithi144.stdout:Sep 16 08:29:20 smithi144 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-7[59146]: 2024-09-16T08:29:20.213+0000 7f4a9aa4d640 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-09-16T08:28:37.859409+0000 front 2024-09-16T08:28:37.859423+0000 (oldest deadline 2024-09-16T08:29:03.759284+0000) 2024-09-16T08:29:20.446 INFO:journalctl@ceph.osd.7.smithi144.stdout:Sep 16 08:29:20 smithi144 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-7[59146]: 2024-09-16T08:29:20.213+0000 7f4a9aa4d640 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.28:6814 osd.1 since back 2024-09-16T08:28:43.760028+0000 front 2024-09-16T08:28:43.759835+0000 (oldest deadline 2024-09-16T08:29:08.459749+0000) 2024-09-16T08:29:20.446 INFO:journalctl@ceph.osd.7.smithi144.stdout:Sep 16 08:29:20 smithi144 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-7[59146]: 2024-09-16T08:29:20.213+0000 7f4a9aa4d640 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.28:6822 osd.2 since back 2024-09-16T08:28:53.160694+0000 front 2024-09-16T08:28:53.160755+0000 (oldest deadline 2024-09-16T08:29:15.460707+0000) 2024-09-16T08:29:20.446 INFO:journalctl@ceph.osd.5.smithi144.stdout:Sep 16 08:29:20 smithi144 podman[69052]: 2024-09-16 08:29:20.065698146 +0000 UTC m=+0.015684975 image pull 425e316ba8ebf74f04aa3640f7deb4fa3f293a78d0c8d097cb679eaf86ddca6c quay.ceph.io/ceph-ci/ceph@sha256:f7ad55f9ccd17ba66aadb19b4d291e67b60ea24e5d8a0c0e92941fd59c81e4ce 2024-09-16T08:29:20.447 INFO:journalctl@ceph.osd.5.smithi144.stdout:Sep 16 08:29:20 smithi144 podman[69052]: 2024-09-16 08:29:20.169285382 +0000 UTC m=+0.119272201 container create f65c5943ed4d12d0868b942a54ab448c6d2ee455deb32994a142f447c35fde63 (image=quay.ceph.io/ceph-ci/ceph@sha256:f7ad55f9ccd17ba66aadb19b4d291e67b60ea24e5d8a0c0e92941fd59c81e4ce, name=ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-5-deactivate, GIT_CLEAN=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, RELEASE=wip-guits-main-2024-09-13-1248-8293d73, org.label-schema.build-date=20240731) 2024-09-16T08:29:20.447 INFO:journalctl@ceph.osd.5.smithi144.stdout:Sep 16 08:29:20 smithi144 podman[69052]: 2024-09-16 08:29:20.371710827 +0000 UTC m=+0.321697647 container init f65c5943ed4d12d0868b942a54ab448c6d2ee455deb32994a142f447c35fde63 (image=quay.ceph.io/ceph-ci/ceph@sha256:f7ad55f9ccd17ba66aadb19b4d291e67b60ea24e5d8a0c0e92941fd59c81e4ce, name=ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-5-deactivate, ceph=True, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, RELEASE=wip-guits-main-2024-09-13-1248-8293d73) 2024-09-16T08:29:20.447 INFO:journalctl@ceph.osd.5.smithi144.stdout:Sep 16 08:29:20 smithi144 podman[69052]: 2024-09-16 08:29:20.375082907 +0000 UTC m=+0.325069730 container start f65c5943ed4d12d0868b942a54ab448c6d2ee455deb32994a142f447c35fde63 (image=quay.ceph.io/ceph-ci/ceph@sha256:f7ad55f9ccd17ba66aadb19b4d291e67b60ea24e5d8a0c0e92941fd59c81e4ce, name=ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-5-deactivate, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, ceph=True, org.label-schema.schema-version=1.0, GIT_CLEAN=True, RELEASE=wip-guits-main-2024-09-13-1248-8293d73, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux ) 2024-09-16T08:29:20.447 INFO:journalctl@ceph.osd.5.smithi144.stdout:Sep 16 08:29:20 smithi144 podman[69052]: 2024-09-16 08:29:20.446704726 +0000 UTC m=+0.396691556 container attach f65c5943ed4d12d0868b942a54ab448c6d2ee455deb32994a142f447c35fde63 (image=quay.ceph.io/ceph-ci/ceph@sha256:f7ad55f9ccd17ba66aadb19b4d291e67b60ea24e5d8a0c0e92941fd59c81e4ce, name=ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-5-deactivate, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, GIT_CLEAN=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, RELEASE=wip-guits-main-2024-09-13-1248-8293d73, ceph=True) 2024-09-16T08:29:20.832 INFO:journalctl@ceph.osd.5.smithi144.stdout:Sep 16 08:29:20 smithi144 podman[69052]: 2024-09-16 08:29:20.569729319 +0000 UTC m=+0.519716135 container died f65c5943ed4d12d0868b942a54ab448c6d2ee455deb32994a142f447c35fde63 (image=quay.ceph.io/ceph-ci/ceph@sha256:f7ad55f9ccd17ba66aadb19b4d291e67b60ea24e5d8a0c0e92941fd59c81e4ce, name=ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-5-deactivate, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, RELEASE=wip-guits-main-2024-09-13-1248-8293d73, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, io.buildah.version=1.37.2) 2024-09-16T08:29:20.832 INFO:journalctl@ceph.osd.6.smithi144.stdout:Sep 16 08:29:20 smithi144 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-6[57105]: 2024-09-16T08:29:20.550+0000 7fc05286c640 -1 osd.6 158 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-09-16T08:28:38.187159+0000 front 2024-09-16T08:28:38.187196+0000 (oldest deadline 2024-09-16T08:29:02.887071+0000) 2024-09-16T08:29:20.832 INFO:journalctl@ceph.osd.6.smithi144.stdout:Sep 16 08:29:20 smithi144 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-6[57105]: 2024-09-16T08:29:20.550+0000 7fc05286c640 -1 osd.6 158 heartbeat_check: no reply from 172.21.15.28:6814 osd.1 since back 2024-09-16T08:28:46.789028+0000 front 2024-09-16T08:28:46.789038+0000 (oldest deadline 2024-09-16T08:29:07.888859+0000) 2024-09-16T08:29:20.832 INFO:journalctl@ceph.osd.6.smithi144.stdout:Sep 16 08:29:20 smithi144 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-6[57105]: 2024-09-16T08:29:20.550+0000 7fc05286c640 -1 osd.6 158 heartbeat_check: no reply from 172.21.15.28:6822 osd.2 since back 2024-09-16T08:28:53.590485+0000 front 2024-09-16T08:28:53.590400+0000 (oldest deadline 2024-09-16T08:29:19.490381+0000) 2024-09-16T08:29:21.127 DEBUG:teuthology.orchestra.run.smithi144:> sudo pkill -f 'journalctl -f -n 0 -u ceph-5622b2fe-7402-11ef-bceb-c7b262605968@osd.5.service' 2024-09-16T08:29:21.167 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-16T08:29:21.167 INFO:tasks.cephadm.osd.5:Stopped osd.5 2024-09-16T08:29:21.167 INFO:tasks.cephadm.osd.6:Stopping osd.6... 2024-09-16T08:29:21.168 DEBUG:teuthology.orchestra.run.smithi144:> sudo systemctl stop ceph-5622b2fe-7402-11ef-bceb-c7b262605968@osd.6 2024-09-16T08:29:21.496 INFO:journalctl@ceph.osd.6.smithi144.stdout:Sep 16 08:29:21 smithi144 systemd[1]: Stopping Ceph osd.6 for 5622b2fe-7402-11ef-bceb-c7b262605968... 2024-09-16T08:29:21.496 INFO:journalctl@ceph.osd.7.smithi144.stdout:Sep 16 08:29:21 smithi144 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-7[59146]: 2024-09-16T08:29:21.204+0000 7f4a9aa4d640 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-09-16T08:28:37.859409+0000 front 2024-09-16T08:28:37.859423+0000 (oldest deadline 2024-09-16T08:29:03.759284+0000) 2024-09-16T08:29:21.496 INFO:journalctl@ceph.osd.7.smithi144.stdout:Sep 16 08:29:21 smithi144 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-7[59146]: 2024-09-16T08:29:21.204+0000 7f4a9aa4d640 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.28:6814 osd.1 since back 2024-09-16T08:28:43.760028+0000 front 2024-09-16T08:28:43.759835+0000 (oldest deadline 2024-09-16T08:29:08.459749+0000) 2024-09-16T08:29:21.497 INFO:journalctl@ceph.osd.7.smithi144.stdout:Sep 16 08:29:21 smithi144 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-7[59146]: 2024-09-16T08:29:21.204+0000 7f4a9aa4d640 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.28:6822 osd.2 since back 2024-09-16T08:28:53.160694+0000 front 2024-09-16T08:28:53.160755+0000 (oldest deadline 2024-09-16T08:29:15.460707+0000) 2024-09-16T08:29:21.832 INFO:journalctl@ceph.osd.6.smithi144.stdout:Sep 16 08:29:21 smithi144 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-6[57105]: 2024-09-16T08:29:21.494+0000 7fc05a25f640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.6 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-16T08:29:21.832 INFO:journalctl@ceph.osd.6.smithi144.stdout:Sep 16 08:29:21 smithi144 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-6[57105]: 2024-09-16T08:29:21.494+0000 7fc05a25f640 -1 osd.6 158 *** Got signal Terminated *** 2024-09-16T08:29:21.832 INFO:journalctl@ceph.osd.6.smithi144.stdout:Sep 16 08:29:21 smithi144 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-6[57105]: 2024-09-16T08:29:21.494+0000 7fc05a25f640 -1 osd.6 158 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-16T08:29:21.832 INFO:journalctl@ceph.osd.6.smithi144.stdout:Sep 16 08:29:21 smithi144 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-6[57105]: 2024-09-16T08:29:21.565+0000 7fc05286c640 -1 osd.6 158 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-09-16T08:28:38.187159+0000 front 2024-09-16T08:28:38.187196+0000 (oldest deadline 2024-09-16T08:29:02.887071+0000) 2024-09-16T08:29:21.832 INFO:journalctl@ceph.osd.6.smithi144.stdout:Sep 16 08:29:21 smithi144 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-6[57105]: 2024-09-16T08:29:21.565+0000 7fc05286c640 -1 osd.6 158 heartbeat_check: no reply from 172.21.15.28:6814 osd.1 since back 2024-09-16T08:28:46.789028+0000 front 2024-09-16T08:28:46.789038+0000 (oldest deadline 2024-09-16T08:29:07.888859+0000) 2024-09-16T08:29:21.832 INFO:journalctl@ceph.osd.6.smithi144.stdout:Sep 16 08:29:21 smithi144 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-6[57105]: 2024-09-16T08:29:21.565+0000 7fc05286c640 -1 osd.6 158 heartbeat_check: no reply from 172.21.15.28:6822 osd.2 since back 2024-09-16T08:28:53.590485+0000 front 2024-09-16T08:28:53.590400+0000 (oldest deadline 2024-09-16T08:29:19.490381+0000) 2024-09-16T08:29:22.538 INFO:journalctl@ceph.osd.7.smithi144.stdout:Sep 16 08:29:22 smithi144 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-7[59146]: 2024-09-16T08:29:22.250+0000 7f4a9aa4d640 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-09-16T08:28:37.859409+0000 front 2024-09-16T08:28:37.859423+0000 (oldest deadline 2024-09-16T08:29:03.759284+0000) 2024-09-16T08:29:22.538 INFO:journalctl@ceph.osd.7.smithi144.stdout:Sep 16 08:29:22 smithi144 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-7[59146]: 2024-09-16T08:29:22.250+0000 7f4a9aa4d640 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.28:6814 osd.1 since back 2024-09-16T08:28:43.760028+0000 front 2024-09-16T08:28:43.759835+0000 (oldest deadline 2024-09-16T08:29:08.459749+0000) 2024-09-16T08:29:22.538 INFO:journalctl@ceph.osd.7.smithi144.stdout:Sep 16 08:29:22 smithi144 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-7[59146]: 2024-09-16T08:29:22.250+0000 7f4a9aa4d640 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.28:6822 osd.2 since back 2024-09-16T08:28:53.160694+0000 front 2024-09-16T08:28:53.160755+0000 (oldest deadline 2024-09-16T08:29:15.460707+0000) 2024-09-16T08:29:22.832 INFO:journalctl@ceph.osd.6.smithi144.stdout:Sep 16 08:29:22 smithi144 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-6[57105]: 2024-09-16T08:29:22.537+0000 7fc05286c640 -1 osd.6 158 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-09-16T08:28:38.187159+0000 front 2024-09-16T08:28:38.187196+0000 (oldest deadline 2024-09-16T08:29:02.887071+0000) 2024-09-16T08:29:22.832 INFO:journalctl@ceph.osd.6.smithi144.stdout:Sep 16 08:29:22 smithi144 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-6[57105]: 2024-09-16T08:29:22.537+0000 7fc05286c640 -1 osd.6 158 heartbeat_check: no reply from 172.21.15.28:6814 osd.1 since back 2024-09-16T08:28:46.789028+0000 front 2024-09-16T08:28:46.789038+0000 (oldest deadline 2024-09-16T08:29:07.888859+0000) 2024-09-16T08:29:22.832 INFO:journalctl@ceph.osd.6.smithi144.stdout:Sep 16 08:29:22 smithi144 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-6[57105]: 2024-09-16T08:29:22.537+0000 7fc05286c640 -1 osd.6 158 heartbeat_check: no reply from 172.21.15.28:6822 osd.2 since back 2024-09-16T08:28:53.590485+0000 front 2024-09-16T08:28:53.590400+0000 (oldest deadline 2024-09-16T08:29:19.490381+0000) 2024-09-16T08:29:23.582 INFO:journalctl@ceph.osd.7.smithi144.stdout:Sep 16 08:29:23 smithi144 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-7[59146]: 2024-09-16T08:29:23.240+0000 7f4a9aa4d640 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-09-16T08:28:37.859409+0000 front 2024-09-16T08:28:37.859423+0000 (oldest deadline 2024-09-16T08:29:03.759284+0000) 2024-09-16T08:29:23.582 INFO:journalctl@ceph.osd.7.smithi144.stdout:Sep 16 08:29:23 smithi144 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-7[59146]: 2024-09-16T08:29:23.240+0000 7f4a9aa4d640 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.28:6814 osd.1 since back 2024-09-16T08:28:43.760028+0000 front 2024-09-16T08:28:43.759835+0000 (oldest deadline 2024-09-16T08:29:08.459749+0000) 2024-09-16T08:29:23.582 INFO:journalctl@ceph.osd.7.smithi144.stdout:Sep 16 08:29:23 smithi144 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-7[59146]: 2024-09-16T08:29:23.240+0000 7f4a9aa4d640 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.28:6822 osd.2 since back 2024-09-16T08:28:53.160694+0000 front 2024-09-16T08:28:53.160755+0000 (oldest deadline 2024-09-16T08:29:15.460707+0000) 2024-09-16T08:29:24.082 INFO:journalctl@ceph.osd.6.smithi144.stdout:Sep 16 08:29:23 smithi144 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-6[57105]: 2024-09-16T08:29:23.588+0000 7fc05286c640 -1 osd.6 158 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-09-16T08:28:38.187159+0000 front 2024-09-16T08:28:38.187196+0000 (oldest deadline 2024-09-16T08:29:02.887071+0000) 2024-09-16T08:29:24.082 INFO:journalctl@ceph.osd.6.smithi144.stdout:Sep 16 08:29:23 smithi144 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-6[57105]: 2024-09-16T08:29:23.588+0000 7fc05286c640 -1 osd.6 158 heartbeat_check: no reply from 172.21.15.28:6814 osd.1 since back 2024-09-16T08:28:46.789028+0000 front 2024-09-16T08:28:46.789038+0000 (oldest deadline 2024-09-16T08:29:07.888859+0000) 2024-09-16T08:29:24.082 INFO:journalctl@ceph.osd.6.smithi144.stdout:Sep 16 08:29:23 smithi144 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-6[57105]: 2024-09-16T08:29:23.588+0000 7fc05286c640 -1 osd.6 158 heartbeat_check: no reply from 172.21.15.28:6822 osd.2 since back 2024-09-16T08:28:53.590485+0000 front 2024-09-16T08:28:53.590400+0000 (oldest deadline 2024-09-16T08:29:19.490381+0000) 2024-09-16T08:29:24.541 INFO:journalctl@ceph.osd.7.smithi144.stdout:Sep 16 08:29:24 smithi144 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-7[59146]: 2024-09-16T08:29:24.206+0000 7f4a9aa4d640 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-09-16T08:28:37.859409+0000 front 2024-09-16T08:28:37.859423+0000 (oldest deadline 2024-09-16T08:29:03.759284+0000) 2024-09-16T08:29:24.541 INFO:journalctl@ceph.osd.7.smithi144.stdout:Sep 16 08:29:24 smithi144 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-7[59146]: 2024-09-16T08:29:24.206+0000 7f4a9aa4d640 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.28:6814 osd.1 since back 2024-09-16T08:28:43.760028+0000 front 2024-09-16T08:28:43.759835+0000 (oldest deadline 2024-09-16T08:29:08.459749+0000) 2024-09-16T08:29:24.541 INFO:journalctl@ceph.osd.7.smithi144.stdout:Sep 16 08:29:24 smithi144 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-7[59146]: 2024-09-16T08:29:24.206+0000 7f4a9aa4d640 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.28:6822 osd.2 since back 2024-09-16T08:28:53.160694+0000 front 2024-09-16T08:28:53.160755+0000 (oldest deadline 2024-09-16T08:29:15.460707+0000) 2024-09-16T08:29:24.831 INFO:journalctl@ceph.osd.6.smithi144.stdout:Sep 16 08:29:24 smithi144 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-6[57105]: 2024-09-16T08:29:24.540+0000 7fc05286c640 -1 osd.6 158 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-09-16T08:28:38.187159+0000 front 2024-09-16T08:28:38.187196+0000 (oldest deadline 2024-09-16T08:29:02.887071+0000) 2024-09-16T08:29:24.832 INFO:journalctl@ceph.osd.6.smithi144.stdout:Sep 16 08:29:24 smithi144 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-6[57105]: 2024-09-16T08:29:24.540+0000 7fc05286c640 -1 osd.6 158 heartbeat_check: no reply from 172.21.15.28:6814 osd.1 since back 2024-09-16T08:28:46.789028+0000 front 2024-09-16T08:28:46.789038+0000 (oldest deadline 2024-09-16T08:29:07.888859+0000) 2024-09-16T08:29:24.832 INFO:journalctl@ceph.osd.6.smithi144.stdout:Sep 16 08:29:24 smithi144 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-6[57105]: 2024-09-16T08:29:24.540+0000 7fc05286c640 -1 osd.6 158 heartbeat_check: no reply from 172.21.15.28:6822 osd.2 since back 2024-09-16T08:28:53.590485+0000 front 2024-09-16T08:28:53.590400+0000 (oldest deadline 2024-09-16T08:29:19.490381+0000) 2024-09-16T08:29:25.519 INFO:journalctl@ceph.osd.7.smithi144.stdout:Sep 16 08:29:25 smithi144 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-7[59146]: 2024-09-16T08:29:25.210+0000 7f4a9aa4d640 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-09-16T08:28:37.859409+0000 front 2024-09-16T08:28:37.859423+0000 (oldest deadline 2024-09-16T08:29:03.759284+0000) 2024-09-16T08:29:25.520 INFO:journalctl@ceph.osd.7.smithi144.stdout:Sep 16 08:29:25 smithi144 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-7[59146]: 2024-09-16T08:29:25.210+0000 7f4a9aa4d640 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.28:6814 osd.1 since back 2024-09-16T08:28:43.760028+0000 front 2024-09-16T08:28:43.759835+0000 (oldest deadline 2024-09-16T08:29:08.459749+0000) 2024-09-16T08:29:25.520 INFO:journalctl@ceph.osd.7.smithi144.stdout:Sep 16 08:29:25 smithi144 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-7[59146]: 2024-09-16T08:29:25.210+0000 7f4a9aa4d640 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.28:6822 osd.2 since back 2024-09-16T08:28:53.160694+0000 front 2024-09-16T08:28:53.160755+0000 (oldest deadline 2024-09-16T08:29:15.460707+0000) 2024-09-16T08:29:25.520 INFO:journalctl@ceph.osd.7.smithi144.stdout:Sep 16 08:29:25 smithi144 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-7[59146]: 2024-09-16T08:29:25.210+0000 7f4a9aa4d640 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.28:6830 osd.3 since back 2024-09-16T08:29:00.062137+0000 front 2024-09-16T08:29:00.062147+0000 (oldest deadline 2024-09-16T08:29:24.762067+0000) 2024-09-16T08:29:25.832 INFO:journalctl@ceph.osd.6.smithi144.stdout:Sep 16 08:29:25 smithi144 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-6[57105]: 2024-09-16T08:29:25.519+0000 7fc05286c640 -1 osd.6 158 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-09-16T08:28:38.187159+0000 front 2024-09-16T08:28:38.187196+0000 (oldest deadline 2024-09-16T08:29:02.887071+0000) 2024-09-16T08:29:25.832 INFO:journalctl@ceph.osd.6.smithi144.stdout:Sep 16 08:29:25 smithi144 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-6[57105]: 2024-09-16T08:29:25.519+0000 7fc05286c640 -1 osd.6 158 heartbeat_check: no reply from 172.21.15.28:6814 osd.1 since back 2024-09-16T08:28:46.789028+0000 front 2024-09-16T08:28:46.789038+0000 (oldest deadline 2024-09-16T08:29:07.888859+0000) 2024-09-16T08:29:25.832 INFO:journalctl@ceph.osd.6.smithi144.stdout:Sep 16 08:29:25 smithi144 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-6[57105]: 2024-09-16T08:29:25.519+0000 7fc05286c640 -1 osd.6 158 heartbeat_check: no reply from 172.21.15.28:6822 osd.2 since back 2024-09-16T08:28:53.590485+0000 front 2024-09-16T08:28:53.590400+0000 (oldest deadline 2024-09-16T08:29:19.490381+0000) 2024-09-16T08:29:25.832 INFO:journalctl@ceph.osd.6.smithi144.stdout:Sep 16 08:29:25 smithi144 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-6[57105]: 2024-09-16T08:29:25.519+0000 7fc05286c640 -1 osd.6 158 heartbeat_check: no reply from 172.21.15.28:6830 osd.3 since back 2024-09-16T08:29:01.791482+0000 front 2024-09-16T08:29:01.791537+0000 (oldest deadline 2024-09-16T08:29:24.691286+0000) 2024-09-16T08:29:26.582 INFO:journalctl@ceph.osd.7.smithi144.stdout:Sep 16 08:29:26 smithi144 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-7[59146]: 2024-09-16T08:29:26.161+0000 7f4a9aa4d640 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-09-16T08:28:37.859409+0000 front 2024-09-16T08:28:37.859423+0000 (oldest deadline 2024-09-16T08:29:03.759284+0000) 2024-09-16T08:29:26.582 INFO:journalctl@ceph.osd.7.smithi144.stdout:Sep 16 08:29:26 smithi144 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-7[59146]: 2024-09-16T08:29:26.161+0000 7f4a9aa4d640 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.28:6814 osd.1 since back 2024-09-16T08:28:43.760028+0000 front 2024-09-16T08:28:43.759835+0000 (oldest deadline 2024-09-16T08:29:08.459749+0000) 2024-09-16T08:29:26.582 INFO:journalctl@ceph.osd.7.smithi144.stdout:Sep 16 08:29:26 smithi144 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-7[59146]: 2024-09-16T08:29:26.161+0000 7f4a9aa4d640 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.28:6822 osd.2 since back 2024-09-16T08:28:53.160694+0000 front 2024-09-16T08:28:53.160755+0000 (oldest deadline 2024-09-16T08:29:15.460707+0000) 2024-09-16T08:29:26.582 INFO:journalctl@ceph.osd.7.smithi144.stdout:Sep 16 08:29:26 smithi144 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-7[59146]: 2024-09-16T08:29:26.161+0000 7f4a9aa4d640 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.28:6830 osd.3 since back 2024-09-16T08:29:00.062137+0000 front 2024-09-16T08:29:00.062147+0000 (oldest deadline 2024-09-16T08:29:24.762067+0000) 2024-09-16T08:29:27.278 INFO:journalctl@ceph.osd.6.smithi144.stdout:Sep 16 08:29:26 smithi144 podman[69155]: 2024-09-16 08:29:26.993930737 +0000 UTC m=+5.596121521 container died b301bf0731f04bbacf337c0c418bc53a126ea72faaf3caf62c415f973fa701e1 (image=quay.ceph.io/ceph-ci/ceph@sha256:f7ad55f9ccd17ba66aadb19b4d291e67b60ea24e5d8a0c0e92941fd59c81e4ce, name=ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-6, RELEASE=wip-guits-main-2024-09-13-1248-8293d73, io.buildah.version=1.37.2, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0) 2024-09-16T08:29:27.278 INFO:journalctl@ceph.osd.7.smithi144.stdout:Sep 16 08:29:27 smithi144 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-7[59146]: 2024-09-16T08:29:27.115+0000 7f4a9aa4d640 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-09-16T08:28:37.859409+0000 front 2024-09-16T08:28:37.859423+0000 (oldest deadline 2024-09-16T08:29:03.759284+0000) 2024-09-16T08:29:27.278 INFO:journalctl@ceph.osd.7.smithi144.stdout:Sep 16 08:29:27 smithi144 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-7[59146]: 2024-09-16T08:29:27.115+0000 7f4a9aa4d640 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.28:6814 osd.1 since back 2024-09-16T08:28:43.760028+0000 front 2024-09-16T08:28:43.759835+0000 (oldest deadline 2024-09-16T08:29:08.459749+0000) 2024-09-16T08:29:27.278 INFO:journalctl@ceph.osd.7.smithi144.stdout:Sep 16 08:29:27 smithi144 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-7[59146]: 2024-09-16T08:29:27.115+0000 7f4a9aa4d640 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.28:6822 osd.2 since back 2024-09-16T08:28:53.160694+0000 front 2024-09-16T08:28:53.160755+0000 (oldest deadline 2024-09-16T08:29:15.460707+0000) 2024-09-16T08:29:27.279 INFO:journalctl@ceph.osd.7.smithi144.stdout:Sep 16 08:29:27 smithi144 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-7[59146]: 2024-09-16T08:29:27.115+0000 7f4a9aa4d640 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.28:6830 osd.3 since back 2024-09-16T08:29:00.062137+0000 front 2024-09-16T08:29:00.062147+0000 (oldest deadline 2024-09-16T08:29:24.762067+0000) 2024-09-16T08:29:27.582 INFO:journalctl@ceph.osd.6.smithi144.stdout:Sep 16 08:29:27 smithi144 podman[69155]: 2024-09-16 08:29:27.370033059 +0000 UTC m=+5.972223841 container cleanup b301bf0731f04bbacf337c0c418bc53a126ea72faaf3caf62c415f973fa701e1 (image=quay.ceph.io/ceph-ci/ceph@sha256:f7ad55f9ccd17ba66aadb19b4d291e67b60ea24e5d8a0c0e92941fd59c81e4ce, name=ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-6, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, ceph=True, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, RELEASE=wip-guits-main-2024-09-13-1248-8293d73, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_CLEAN=True, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-16T08:29:27.582 INFO:journalctl@ceph.osd.6.smithi144.stdout:Sep 16 08:29:27 smithi144 bash[69155]: ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-6 2024-09-16T08:29:27.939 INFO:journalctl@ceph.osd.6.smithi144.stdout:Sep 16 08:29:27 smithi144 podman[69172]: 2024-09-16 08:29:27.612894934 +0000 UTC m=+0.621342100 container remove b301bf0731f04bbacf337c0c418bc53a126ea72faaf3caf62c415f973fa701e1 (image=quay.ceph.io/ceph-ci/ceph@sha256:f7ad55f9ccd17ba66aadb19b4d291e67b60ea24e5d8a0c0e92941fd59c81e4ce, name=ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-6, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_CLEAN=True, io.buildah.version=1.37.2, RELEASE=wip-guits-main-2024-09-13-1248-8293d73, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, ceph=True, org.label-schema.license=GPLv2) 2024-09-16T08:29:28.249 INFO:journalctl@ceph.osd.7.smithi144.stdout:Sep 16 08:29:28 smithi144 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-7[59146]: 2024-09-16T08:29:28.114+0000 7f4a9aa4d640 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-09-16T08:28:37.859409+0000 front 2024-09-16T08:28:37.859423+0000 (oldest deadline 2024-09-16T08:29:03.759284+0000) 2024-09-16T08:29:28.249 INFO:journalctl@ceph.osd.7.smithi144.stdout:Sep 16 08:29:28 smithi144 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-7[59146]: 2024-09-16T08:29:28.114+0000 7f4a9aa4d640 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.28:6814 osd.1 since back 2024-09-16T08:28:43.760028+0000 front 2024-09-16T08:28:43.759835+0000 (oldest deadline 2024-09-16T08:29:08.459749+0000) 2024-09-16T08:29:28.249 INFO:journalctl@ceph.osd.7.smithi144.stdout:Sep 16 08:29:28 smithi144 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-7[59146]: 2024-09-16T08:29:28.114+0000 7f4a9aa4d640 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.28:6822 osd.2 since back 2024-09-16T08:28:53.160694+0000 front 2024-09-16T08:28:53.160755+0000 (oldest deadline 2024-09-16T08:29:15.460707+0000) 2024-09-16T08:29:28.249 INFO:journalctl@ceph.osd.7.smithi144.stdout:Sep 16 08:29:28 smithi144 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-7[59146]: 2024-09-16T08:29:28.114+0000 7f4a9aa4d640 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.28:6830 osd.3 since back 2024-09-16T08:29:00.062137+0000 front 2024-09-16T08:29:00.062147+0000 (oldest deadline 2024-09-16T08:29:24.762067+0000) 2024-09-16T08:29:28.249 INFO:journalctl@ceph.osd.6.smithi144.stdout:Sep 16 08:29:27 smithi144 podman[69261]: 2024-09-16 08:29:27.842406482 +0000 UTC m=+0.015749914 image pull 425e316ba8ebf74f04aa3640f7deb4fa3f293a78d0c8d097cb679eaf86ddca6c quay.ceph.io/ceph-ci/ceph@sha256:f7ad55f9ccd17ba66aadb19b4d291e67b60ea24e5d8a0c0e92941fd59c81e4ce 2024-09-16T08:29:28.249 INFO:journalctl@ceph.osd.6.smithi144.stdout:Sep 16 08:29:27 smithi144 podman[69261]: 2024-09-16 08:29:27.955200606 +0000 UTC m=+0.128544031 container create 24fdc4a14a0e8c145503c1c330d7d3b527a18ed2422d184f7668648305fa7135 (image=quay.ceph.io/ceph-ci/ceph@sha256:f7ad55f9ccd17ba66aadb19b4d291e67b60ea24e5d8a0c0e92941fd59c81e4ce, name=ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-6-deactivate, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, ceph=True, RELEASE=wip-guits-main-2024-09-13-1248-8293d73, maintainer=Guillaume Abrioux ) 2024-09-16T08:29:28.250 INFO:journalctl@ceph.osd.6.smithi144.stdout:Sep 16 08:29:28 smithi144 podman[69261]: 2024-09-16 08:29:28.166446529 +0000 UTC m=+0.339789955 container init 24fdc4a14a0e8c145503c1c330d7d3b527a18ed2422d184f7668648305fa7135 (image=quay.ceph.io/ceph-ci/ceph@sha256:f7ad55f9ccd17ba66aadb19b4d291e67b60ea24e5d8a0c0e92941fd59c81e4ce, name=ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-6-deactivate, ceph=True, RELEASE=wip-guits-main-2024-09-13-1248-8293d73, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_CLEAN=True, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS) 2024-09-16T08:29:28.250 INFO:journalctl@ceph.osd.6.smithi144.stdout:Sep 16 08:29:28 smithi144 podman[69261]: 2024-09-16 08:29:28.169722032 +0000 UTC m=+0.343065468 container start 24fdc4a14a0e8c145503c1c330d7d3b527a18ed2422d184f7668648305fa7135 (image=quay.ceph.io/ceph-ci/ceph@sha256:f7ad55f9ccd17ba66aadb19b4d291e67b60ea24e5d8a0c0e92941fd59c81e4ce, name=ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-6-deactivate, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, RELEASE=wip-guits-main-2024-09-13-1248-8293d73, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, ceph=True) 2024-09-16T08:29:28.250 INFO:journalctl@ceph.osd.6.smithi144.stdout:Sep 16 08:29:28 smithi144 podman[69261]: 2024-09-16 08:29:28.249471138 +0000 UTC m=+0.422814567 container attach 24fdc4a14a0e8c145503c1c330d7d3b527a18ed2422d184f7668648305fa7135 (image=quay.ceph.io/ceph-ci/ceph@sha256:f7ad55f9ccd17ba66aadb19b4d291e67b60ea24e5d8a0c0e92941fd59c81e4ce, name=ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-6-deactivate, org.label-schema.build-date=20240731, RELEASE=wip-guits-main-2024-09-13-1248-8293d73, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_CLEAN=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux ) 2024-09-16T08:29:28.582 INFO:journalctl@ceph.osd.6.smithi144.stdout:Sep 16 08:29:28 smithi144 podman[69261]: 2024-09-16 08:29:28.356396759 +0000 UTC m=+0.529740186 container died 24fdc4a14a0e8c145503c1c330d7d3b527a18ed2422d184f7668648305fa7135 (image=quay.ceph.io/ceph-ci/ceph@sha256:f7ad55f9ccd17ba66aadb19b4d291e67b60ea24e5d8a0c0e92941fd59c81e4ce, name=ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-6-deactivate, io.buildah.version=1.37.2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, RELEASE=wip-guits-main-2024-09-13-1248-8293d73, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.license=GPLv2, GIT_CLEAN=True) 2024-09-16T08:29:28.953 DEBUG:teuthology.orchestra.run.smithi144:> sudo pkill -f 'journalctl -f -n 0 -u ceph-5622b2fe-7402-11ef-bceb-c7b262605968@osd.6.service' 2024-09-16T08:29:28.993 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-16T08:29:28.993 INFO:tasks.cephadm.osd.6:Stopped osd.6 2024-09-16T08:29:28.993 INFO:tasks.cephadm.osd.7:Stopping osd.7... 2024-09-16T08:29:28.993 DEBUG:teuthology.orchestra.run.smithi144:> sudo systemctl stop ceph-5622b2fe-7402-11ef-bceb-c7b262605968@osd.7 2024-09-16T08:29:29.323 INFO:journalctl@ceph.osd.7.smithi144.stdout:Sep 16 08:29:29 smithi144 systemd[1]: Stopping Ceph osd.7 for 5622b2fe-7402-11ef-bceb-c7b262605968... 2024-09-16T08:29:29.323 INFO:journalctl@ceph.osd.7.smithi144.stdout:Sep 16 08:29:29 smithi144 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-7[59146]: 2024-09-16T08:29:29.110+0000 7f4a9aa4d640 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-09-16T08:28:37.859409+0000 front 2024-09-16T08:28:37.859423+0000 (oldest deadline 2024-09-16T08:29:03.759284+0000) 2024-09-16T08:29:29.323 INFO:journalctl@ceph.osd.7.smithi144.stdout:Sep 16 08:29:29 smithi144 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-7[59146]: 2024-09-16T08:29:29.110+0000 7f4a9aa4d640 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.28:6814 osd.1 since back 2024-09-16T08:28:43.760028+0000 front 2024-09-16T08:28:43.759835+0000 (oldest deadline 2024-09-16T08:29:08.459749+0000) 2024-09-16T08:29:29.323 INFO:journalctl@ceph.osd.7.smithi144.stdout:Sep 16 08:29:29 smithi144 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-7[59146]: 2024-09-16T08:29:29.110+0000 7f4a9aa4d640 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.28:6822 osd.2 since back 2024-09-16T08:28:53.160694+0000 front 2024-09-16T08:28:53.160755+0000 (oldest deadline 2024-09-16T08:29:15.460707+0000) 2024-09-16T08:29:29.323 INFO:journalctl@ceph.osd.7.smithi144.stdout:Sep 16 08:29:29 smithi144 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-7[59146]: 2024-09-16T08:29:29.110+0000 7f4a9aa4d640 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.28:6830 osd.3 since back 2024-09-16T08:29:00.062137+0000 front 2024-09-16T08:29:00.062147+0000 (oldest deadline 2024-09-16T08:29:24.762067+0000) 2024-09-16T08:29:29.323 INFO:journalctl@ceph.osd.7.smithi144.stdout:Sep 16 08:29:29 smithi144 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-7[59146]: 2024-09-16T08:29:29.322+0000 7f4a9e438640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.7 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-16T08:29:29.323 INFO:journalctl@ceph.osd.7.smithi144.stdout:Sep 16 08:29:29 smithi144 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-7[59146]: 2024-09-16T08:29:29.322+0000 7f4a9e438640 -1 osd.7 158 *** Got signal Terminated *** 2024-09-16T08:29:29.323 INFO:journalctl@ceph.osd.7.smithi144.stdout:Sep 16 08:29:29 smithi144 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-7[59146]: 2024-09-16T08:29:29.322+0000 7f4a9e438640 -1 osd.7 158 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-16T08:29:30.582 INFO:journalctl@ceph.osd.7.smithi144.stdout:Sep 16 08:29:30 smithi144 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-7[59146]: 2024-09-16T08:29:30.109+0000 7f4a9aa4d640 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-09-16T08:28:37.859409+0000 front 2024-09-16T08:28:37.859423+0000 (oldest deadline 2024-09-16T08:29:03.759284+0000) 2024-09-16T08:29:30.582 INFO:journalctl@ceph.osd.7.smithi144.stdout:Sep 16 08:29:30 smithi144 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-7[59146]: 2024-09-16T08:29:30.109+0000 7f4a9aa4d640 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.28:6814 osd.1 since back 2024-09-16T08:28:43.760028+0000 front 2024-09-16T08:28:43.759835+0000 (oldest deadline 2024-09-16T08:29:08.459749+0000) 2024-09-16T08:29:30.582 INFO:journalctl@ceph.osd.7.smithi144.stdout:Sep 16 08:29:30 smithi144 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-7[59146]: 2024-09-16T08:29:30.109+0000 7f4a9aa4d640 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.28:6822 osd.2 since back 2024-09-16T08:28:53.160694+0000 front 2024-09-16T08:28:53.160755+0000 (oldest deadline 2024-09-16T08:29:15.460707+0000) 2024-09-16T08:29:30.582 INFO:journalctl@ceph.osd.7.smithi144.stdout:Sep 16 08:29:30 smithi144 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-7[59146]: 2024-09-16T08:29:30.109+0000 7f4a9aa4d640 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.28:6830 osd.3 since back 2024-09-16T08:29:00.062137+0000 front 2024-09-16T08:29:00.062147+0000 (oldest deadline 2024-09-16T08:29:24.762067+0000) 2024-09-16T08:29:31.582 INFO:journalctl@ceph.osd.7.smithi144.stdout:Sep 16 08:29:31 smithi144 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-7[59146]: 2024-09-16T08:29:31.105+0000 7f4a9aa4d640 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-09-16T08:28:37.859409+0000 front 2024-09-16T08:28:37.859423+0000 (oldest deadline 2024-09-16T08:29:03.759284+0000) 2024-09-16T08:29:31.582 INFO:journalctl@ceph.osd.7.smithi144.stdout:Sep 16 08:29:31 smithi144 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-7[59146]: 2024-09-16T08:29:31.105+0000 7f4a9aa4d640 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.28:6814 osd.1 since back 2024-09-16T08:28:43.760028+0000 front 2024-09-16T08:28:43.759835+0000 (oldest deadline 2024-09-16T08:29:08.459749+0000) 2024-09-16T08:29:31.582 INFO:journalctl@ceph.osd.7.smithi144.stdout:Sep 16 08:29:31 smithi144 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-7[59146]: 2024-09-16T08:29:31.105+0000 7f4a9aa4d640 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.28:6822 osd.2 since back 2024-09-16T08:28:53.160694+0000 front 2024-09-16T08:28:53.160755+0000 (oldest deadline 2024-09-16T08:29:15.460707+0000) 2024-09-16T08:29:31.582 INFO:journalctl@ceph.osd.7.smithi144.stdout:Sep 16 08:29:31 smithi144 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-7[59146]: 2024-09-16T08:29:31.105+0000 7f4a9aa4d640 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.28:6830 osd.3 since back 2024-09-16T08:29:00.062137+0000 front 2024-09-16T08:29:00.062147+0000 (oldest deadline 2024-09-16T08:29:24.762067+0000) 2024-09-16T08:29:32.483 INFO:journalctl@ceph.osd.7.smithi144.stdout:Sep 16 08:29:32 smithi144 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-7[59146]: 2024-09-16T08:29:32.105+0000 7f4a9aa4d640 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-09-16T08:28:37.859409+0000 front 2024-09-16T08:28:37.859423+0000 (oldest deadline 2024-09-16T08:29:03.759284+0000) 2024-09-16T08:29:32.483 INFO:journalctl@ceph.osd.7.smithi144.stdout:Sep 16 08:29:32 smithi144 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-7[59146]: 2024-09-16T08:29:32.105+0000 7f4a9aa4d640 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.28:6814 osd.1 since back 2024-09-16T08:28:43.760028+0000 front 2024-09-16T08:28:43.759835+0000 (oldest deadline 2024-09-16T08:29:08.459749+0000) 2024-09-16T08:29:32.484 INFO:journalctl@ceph.osd.7.smithi144.stdout:Sep 16 08:29:32 smithi144 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-7[59146]: 2024-09-16T08:29:32.105+0000 7f4a9aa4d640 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.28:6822 osd.2 since back 2024-09-16T08:28:53.160694+0000 front 2024-09-16T08:28:53.160755+0000 (oldest deadline 2024-09-16T08:29:15.460707+0000) 2024-09-16T08:29:32.484 INFO:journalctl@ceph.osd.7.smithi144.stdout:Sep 16 08:29:32 smithi144 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-7[59146]: 2024-09-16T08:29:32.105+0000 7f4a9aa4d640 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.28:6830 osd.3 since back 2024-09-16T08:29:00.062137+0000 front 2024-09-16T08:29:00.062147+0000 (oldest deadline 2024-09-16T08:29:24.762067+0000) 2024-09-16T08:29:32.484 INFO:journalctl@ceph.osd.7.smithi144.stdout:Sep 16 08:29:32 smithi144 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-7[59146]: 2024-09-16T08:29:32.105+0000 7f4a9aa4d640 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.144:6804 osd.4 since back 2024-09-16T08:29:07.063059+0000 front 2024-09-16T08:29:07.063004+0000 (oldest deadline 2024-09-16T08:29:31.762976+0000) 2024-09-16T08:29:33.332 INFO:journalctl@ceph.osd.7.smithi144.stdout:Sep 16 08:29:33 smithi144 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-7[59146]: 2024-09-16T08:29:33.122+0000 7f4a9aa4d640 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-09-16T08:28:37.859409+0000 front 2024-09-16T08:28:37.859423+0000 (oldest deadline 2024-09-16T08:29:03.759284+0000) 2024-09-16T08:29:33.332 INFO:journalctl@ceph.osd.7.smithi144.stdout:Sep 16 08:29:33 smithi144 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-7[59146]: 2024-09-16T08:29:33.122+0000 7f4a9aa4d640 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.28:6814 osd.1 since back 2024-09-16T08:28:43.760028+0000 front 2024-09-16T08:28:43.759835+0000 (oldest deadline 2024-09-16T08:29:08.459749+0000) 2024-09-16T08:29:33.332 INFO:journalctl@ceph.osd.7.smithi144.stdout:Sep 16 08:29:33 smithi144 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-7[59146]: 2024-09-16T08:29:33.122+0000 7f4a9aa4d640 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.28:6822 osd.2 since back 2024-09-16T08:28:53.160694+0000 front 2024-09-16T08:28:53.160755+0000 (oldest deadline 2024-09-16T08:29:15.460707+0000) 2024-09-16T08:29:33.332 INFO:journalctl@ceph.osd.7.smithi144.stdout:Sep 16 08:29:33 smithi144 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-7[59146]: 2024-09-16T08:29:33.122+0000 7f4a9aa4d640 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.28:6830 osd.3 since back 2024-09-16T08:29:00.062137+0000 front 2024-09-16T08:29:00.062147+0000 (oldest deadline 2024-09-16T08:29:24.762067+0000) 2024-09-16T08:29:33.332 INFO:journalctl@ceph.osd.7.smithi144.stdout:Sep 16 08:29:33 smithi144 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-7[59146]: 2024-09-16T08:29:33.122+0000 7f4a9aa4d640 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.144:6804 osd.4 since back 2024-09-16T08:29:07.063059+0000 front 2024-09-16T08:29:07.063004+0000 (oldest deadline 2024-09-16T08:29:31.762976+0000) 2024-09-16T08:29:34.582 INFO:journalctl@ceph.osd.7.smithi144.stdout:Sep 16 08:29:34 smithi144 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-7[59146]: 2024-09-16T08:29:34.120+0000 7f4a9aa4d640 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-09-16T08:28:37.859409+0000 front 2024-09-16T08:28:37.859423+0000 (oldest deadline 2024-09-16T08:29:03.759284+0000) 2024-09-16T08:29:34.582 INFO:journalctl@ceph.osd.7.smithi144.stdout:Sep 16 08:29:34 smithi144 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-7[59146]: 2024-09-16T08:29:34.120+0000 7f4a9aa4d640 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.28:6814 osd.1 since back 2024-09-16T08:28:43.760028+0000 front 2024-09-16T08:28:43.759835+0000 (oldest deadline 2024-09-16T08:29:08.459749+0000) 2024-09-16T08:29:34.582 INFO:journalctl@ceph.osd.7.smithi144.stdout:Sep 16 08:29:34 smithi144 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-7[59146]: 2024-09-16T08:29:34.120+0000 7f4a9aa4d640 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.28:6822 osd.2 since back 2024-09-16T08:28:53.160694+0000 front 2024-09-16T08:28:53.160755+0000 (oldest deadline 2024-09-16T08:29:15.460707+0000) 2024-09-16T08:29:34.582 INFO:journalctl@ceph.osd.7.smithi144.stdout:Sep 16 08:29:34 smithi144 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-7[59146]: 2024-09-16T08:29:34.120+0000 7f4a9aa4d640 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.28:6830 osd.3 since back 2024-09-16T08:29:00.062137+0000 front 2024-09-16T08:29:00.062147+0000 (oldest deadline 2024-09-16T08:29:24.762067+0000) 2024-09-16T08:29:34.582 INFO:journalctl@ceph.osd.7.smithi144.stdout:Sep 16 08:29:34 smithi144 ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-7[59146]: 2024-09-16T08:29:34.120+0000 7f4a9aa4d640 -1 osd.7 158 heartbeat_check: no reply from 172.21.15.144:6804 osd.4 since back 2024-09-16T08:29:07.063059+0000 front 2024-09-16T08:29:07.063004+0000 (oldest deadline 2024-09-16T08:29:31.762976+0000) 2024-09-16T08:29:35.014 INFO:journalctl@ceph.osd.7.smithi144.stdout:Sep 16 08:29:34 smithi144 podman[69366]: 2024-09-16 08:29:34.75458958 +0000 UTC m=+5.529590408 container died f1a5cbf3a7bd4507c6d1a99600de331ca98754dd71c18ffb754e85900c30bd5e (image=quay.ceph.io/ceph-ci/ceph@sha256:f7ad55f9ccd17ba66aadb19b4d291e67b60ea24e5d8a0c0e92941fd59c81e4ce, name=ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-7, ceph=True, org.label-schema.vendor=CentOS, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.build-date=20240731, GIT_CLEAN=True, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=wip-guits-main-2024-09-13-1248-8293d73) 2024-09-16T08:29:35.332 INFO:journalctl@ceph.osd.7.smithi144.stdout:Sep 16 08:29:35 smithi144 podman[69366]: 2024-09-16 08:29:35.191203681 +0000 UTC m=+5.966204510 container cleanup f1a5cbf3a7bd4507c6d1a99600de331ca98754dd71c18ffb754e85900c30bd5e (image=quay.ceph.io/ceph-ci/ceph@sha256:f7ad55f9ccd17ba66aadb19b4d291e67b60ea24e5d8a0c0e92941fd59c81e4ce, name=ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-7, GIT_CLEAN=True, ceph=True, RELEASE=wip-guits-main-2024-09-13-1248-8293d73, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-16T08:29:35.332 INFO:journalctl@ceph.osd.7.smithi144.stdout:Sep 16 08:29:35 smithi144 bash[69366]: ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-7 2024-09-16T08:29:35.794 INFO:journalctl@ceph.osd.7.smithi144.stdout:Sep 16 08:29:35 smithi144 podman[69394]: 2024-09-16 08:29:35.382211005 +0000 UTC m=+0.625097126 container remove f1a5cbf3a7bd4507c6d1a99600de331ca98754dd71c18ffb754e85900c30bd5e (image=quay.ceph.io/ceph-ci/ceph@sha256:f7ad55f9ccd17ba66aadb19b4d291e67b60ea24e5d8a0c0e92941fd59c81e4ce, name=ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-7, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, RELEASE=wip-guits-main-2024-09-13-1248-8293d73, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965) 2024-09-16T08:29:36.046 INFO:journalctl@ceph.osd.7.smithi144.stdout:Sep 16 08:29:35 smithi144 podman[69483]: 2024-09-16 08:29:35.696987828 +0000 UTC m=+0.015874385 image pull 425e316ba8ebf74f04aa3640f7deb4fa3f293a78d0c8d097cb679eaf86ddca6c quay.ceph.io/ceph-ci/ceph@sha256:f7ad55f9ccd17ba66aadb19b4d291e67b60ea24e5d8a0c0e92941fd59c81e4ce 2024-09-16T08:29:36.046 INFO:journalctl@ceph.osd.7.smithi144.stdout:Sep 16 08:29:35 smithi144 podman[69483]: 2024-09-16 08:29:35.816325635 +0000 UTC m=+0.135212191 container create 194a7fa6e58483b69810e39996f104d0946df2a93dfe67de602790685492a3c9 (image=quay.ceph.io/ceph-ci/ceph@sha256:f7ad55f9ccd17ba66aadb19b4d291e67b60ea24e5d8a0c0e92941fd59c81e4ce, name=ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-7-deactivate, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, ceph=True, RELEASE=wip-guits-main-2024-09-13-1248-8293d73, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_CLEAN=True, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965) 2024-09-16T08:29:36.047 INFO:journalctl@ceph.osd.7.smithi144.stdout:Sep 16 08:29:36 smithi144 podman[69483]: 2024-09-16 08:29:36.043742591 +0000 UTC m=+0.362629160 container init 194a7fa6e58483b69810e39996f104d0946df2a93dfe67de602790685492a3c9 (image=quay.ceph.io/ceph-ci/ceph@sha256:f7ad55f9ccd17ba66aadb19b4d291e67b60ea24e5d8a0c0e92941fd59c81e4ce, name=ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-7-deactivate, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, RELEASE=wip-guits-main-2024-09-13-1248-8293d73, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, ceph=True) 2024-09-16T08:29:36.047 INFO:journalctl@ceph.osd.7.smithi144.stdout:Sep 16 08:29:36 smithi144 podman[69483]: 2024-09-16 08:29:36.04691525 +0000 UTC m=+0.365801815 container start 194a7fa6e58483b69810e39996f104d0946df2a93dfe67de602790685492a3c9 (image=quay.ceph.io/ceph-ci/ceph@sha256:f7ad55f9ccd17ba66aadb19b4d291e67b60ea24e5d8a0c0e92941fd59c81e4ce, name=ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-7-deactivate, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=wip-guits-main-2024-09-13-1248-8293d73, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, GIT_CLEAN=True, GIT_BRANCH=HEAD, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.build-date=20240731) 2024-09-16T08:29:36.332 INFO:journalctl@ceph.osd.7.smithi144.stdout:Sep 16 08:29:36 smithi144 podman[69483]: 2024-09-16 08:29:36.143764815 +0000 UTC m=+0.462651390 container attach 194a7fa6e58483b69810e39996f104d0946df2a93dfe67de602790685492a3c9 (image=quay.ceph.io/ceph-ci/ceph@sha256:f7ad55f9ccd17ba66aadb19b4d291e67b60ea24e5d8a0c0e92941fd59c81e4ce, name=ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-7-deactivate, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, RELEASE=wip-guits-main-2024-09-13-1248-8293d73, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, ceph=True) 2024-09-16T08:29:36.332 INFO:journalctl@ceph.osd.7.smithi144.stdout:Sep 16 08:29:36 smithi144 podman[69483]: 2024-09-16 08:29:36.231922648 +0000 UTC m=+0.550809208 container died 194a7fa6e58483b69810e39996f104d0946df2a93dfe67de602790685492a3c9 (image=quay.ceph.io/ceph-ci/ceph@sha256:f7ad55f9ccd17ba66aadb19b4d291e67b60ea24e5d8a0c0e92941fd59c81e4ce, name=ceph-5622b2fe-7402-11ef-bceb-c7b262605968-osd-7-deactivate, RELEASE=wip-guits-main-2024-09-13-1248-8293d73, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_CLEAN=True, GIT_BRANCH=HEAD) 2024-09-16T08:29:36.913 DEBUG:teuthology.orchestra.run.smithi144:> sudo pkill -f 'journalctl -f -n 0 -u ceph-5622b2fe-7402-11ef-bceb-c7b262605968@osd.7.service' 2024-09-16T08:29:36.956 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-16T08:29:36.956 INFO:tasks.cephadm.osd.7:Stopped osd.7 2024-09-16T08:29:36.956 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 5622b2fe-7402-11ef-bceb-c7b262605968 --force --keep-logs 2024-09-16T08:32:36.854 DEBUG:teuthology.orchestra.run.smithi144:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 5622b2fe-7402-11ef-bceb-c7b262605968 --force --keep-logs 2024-09-16T08:32:58.440 DEBUG:teuthology.orchestra.run.smithi028:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-09-16T08:32:58.477 DEBUG:teuthology.orchestra.run.smithi144:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-09-16T08:32:58.504 INFO:tasks.cephadm:Archiving crash dumps... 2024-09-16T08:32:58.507 DEBUG:teuthology.misc:Transferring archived files from smithi028:/var/lib/ceph/5622b2fe-7402-11ef-bceb-c7b262605968/crash to /home/teuthworker/archive/gabrioux-2024-09-16_07:44:22-orch:cephadm-wip-guits-main-2024-09-13-1248-distro-default-smithi/7906916/remote/smithi028/crash 2024-09-16T08:32:58.508 DEBUG:teuthology.orchestra.run.smithi028:> sudo tar c -f - -C /var/lib/ceph/5622b2fe-7402-11ef-bceb-c7b262605968/crash -- . 2024-09-16T08:32:58.561 INFO:teuthology.orchestra.run.smithi028.stderr:tar: /var/lib/ceph/5622b2fe-7402-11ef-bceb-c7b262605968/crash: Cannot open: No such file or directory 2024-09-16T08:32:58.562 INFO:teuthology.orchestra.run.smithi028.stderr:tar: Error is not recoverable: exiting now 2024-09-16T08:32:58.564 DEBUG:teuthology.misc:Transferring archived files from smithi144:/var/lib/ceph/5622b2fe-7402-11ef-bceb-c7b262605968/crash to /home/teuthworker/archive/gabrioux-2024-09-16_07:44:22-orch:cephadm-wip-guits-main-2024-09-13-1248-distro-default-smithi/7906916/remote/smithi144/crash 2024-09-16T08:32:58.564 DEBUG:teuthology.orchestra.run.smithi144:> sudo tar c -f - -C /var/lib/ceph/5622b2fe-7402-11ef-bceb-c7b262605968/crash -- . 2024-09-16T08:32:58.604 INFO:teuthology.orchestra.run.smithi144.stderr:tar: /var/lib/ceph/5622b2fe-7402-11ef-bceb-c7b262605968/crash: Cannot open: No such file or directory 2024-09-16T08:32:58.604 INFO:teuthology.orchestra.run.smithi144.stderr:tar: Error is not recoverable: exiting now 2024-09-16T08:32:58.605 INFO:tasks.cephadm:Checking cluster log for badness... 2024-09-16T08:32:58.606 DEBUG:teuthology.orchestra.run.smithi028:> sudo egrep '\[ERR\]|\[WRN\]|\[SEC\]' /var/log/ceph/5622b2fe-7402-11ef-bceb-c7b262605968/ceph.log | egrep CEPHADM_ | egrep -v '\(MDS_ALL_DOWN\)' | egrep -v '\(MDS_UP_LESS_THAN_MAX\)' | egrep -v CEPHADM_REFRESH_FAILED | head -n 1 2024-09-16T08:32:58.637 INFO:tasks.cephadm:Compressing logs... 2024-09-16T08:32:58.637 DEBUG:teuthology.orchestra.run.smithi028:> time sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs --max-args=1 --max-procs=0 --verbose -0 --no-run-if-empty -- gzip -5 --verbose -- 2024-09-16T08:32:58.679 DEBUG:teuthology.orchestra.run.smithi144:> time sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs --max-args=1 --max-procs=0 --verbose -0 --no-run-if-empty -- gzip -5 --verbose -- 2024-09-16T08:32:58.703 INFO:teuthology.orchestra.run.smithi144.stderr:find: ‘/var/log/rbd-target-api’: No such file or directory 2024-09-16T08:32:58.703 INFO:teuthology.orchestra.run.smithi144.stderr:gzip -5 --verbose -- /var/log/ceph/5622b2fe-7402-11ef-bceb-c7b262605968/ceph-mon.smithi144.log 2024-09-16T08:32:58.703 INFO:teuthology.orchestra.run.smithi144.stderr:gzip -5 --verbose -- /var/log/ceph/5622b2fe-7402-11ef-bceb-c7b262605968/ceph-osd.7.log 2024-09-16T08:32:58.705 INFO:teuthology.orchestra.run.smithi144.stderr:gzip -5 --verbose -- /var/log/ceph/5622b2fe-7402-11ef-bceb-c7b262605968/ceph-volume.log 2024-09-16T08:32:58.706 INFO:teuthology.orchestra.run.smithi144.stderr:/var/log/ceph/5622b2fe-7402-11ef-bceb-c7b262605968/ceph-mon.smithi144.log: /var/log/ceph/5622b2fe-7402-11ef-bceb-c7b262605968/ceph-osd.7.log: gzip -5 --verbose -- /var/log/ceph/5622b2fe-7402-11ef-bceb-c7b262605968/ceph-osd.4.log 2024-09-16T08:32:58.706 INFO:teuthology.orchestra.run.smithi144.stderr:/var/log/ceph/5622b2fe-7402-11ef-bceb-c7b262605968/ceph-volume.log: gzip -5 --verbose -- /var/log/ceph/5622b2fe-7402-11ef-bceb-c7b262605968/ceph.audit.log 2024-09-16T08:32:58.706 INFO:teuthology.orchestra.run.smithi144.stderr:/var/log/ceph/5622b2fe-7402-11ef-bceb-c7b262605968/ceph-osd.4.log: gzip -5 --verbose -- /var/log/ceph/5622b2fe-7402-11ef-bceb-c7b262605968/ceph-mgr.smithi144.utzszj.log 2024-09-16T08:32:58.706 INFO:teuthology.orchestra.run.smithi144.stderr:/var/log/ceph/5622b2fe-7402-11ef-bceb-c7b262605968/ceph.audit.log: gzip -5 --verbose -- /var/log/ceph/5622b2fe-7402-11ef-bceb-c7b262605968/ceph.cephadm.log 2024-09-16T08:32:58.706 INFO:teuthology.orchestra.run.smithi144.stderr:/var/log/ceph/5622b2fe-7402-11ef-bceb-c7b262605968/ceph-mgr.smithi144.utzszj.log: gzip -5 --verbose -- /var/log/ceph/5622b2fe-7402-11ef-bceb-c7b262605968/ceph-osd.6.log 2024-09-16T08:32:58.707 INFO:teuthology.orchestra.run.smithi144.stderr:/var/log/ceph/5622b2fe-7402-11ef-bceb-c7b262605968/ceph.cephadm.log: gzip -5 --verbose -- /var/log/ceph/5622b2fe-7402-11ef-bceb-c7b262605968/ceph-osd.5.log 2024-09-16T08:32:58.707 INFO:teuthology.orchestra.run.smithi144.stderr:/var/log/ceph/5622b2fe-7402-11ef-bceb-c7b262605968/ceph-osd.6.log: 85.5% -- replaced with /var/log/ceph/5622b2fe-7402-11ef-bceb-c7b262605968/ceph.cephadm.log.gz 2024-09-16T08:32:58.708 INFO:teuthology.orchestra.run.smithi028.stderr:find: ‘/var/log/rbd-target-api’gzip -5 --verbose -- /var/log/ceph/5622b2fe-7402-11ef-bceb-c7b262605968/ceph-osd.0.log 2024-09-16T08:32:58.708 INFO:teuthology.orchestra.run.smithi028.stderr:: No such file or directory 2024-09-16T08:32:58.708 INFO:teuthology.orchestra.run.smithi028.stderr:gzip -5 --verbose -- /var/log/ceph/5622b2fe-7402-11ef-bceb-c7b262605968/ceph-osd.1.log 2024-09-16T08:32:58.708 INFO:teuthology.orchestra.run.smithi028.stderr:gzip -5 --verbose -- /var/log/ceph/5622b2fe-7402-11ef-bceb-c7b262605968/ceph-mon.smithi028.log 2024-09-16T08:32:58.708 INFO:teuthology.orchestra.run.smithi028.stderr:/var/log/ceph/5622b2fe-7402-11ef-bceb-c7b262605968/ceph-osd.0.log: /var/log/ceph/5622b2fe-7402-11ef-bceb-c7b262605968/ceph-osd.1.log: gzip -5 --verbose -- /var/log/ceph/5622b2fe-7402-11ef-bceb-c7b262605968/ceph-volume.log 2024-09-16T08:32:58.708 INFO:teuthology.orchestra.run.smithi028.stderr:/var/log/ceph/5622b2fe-7402-11ef-bceb-c7b262605968/ceph-mon.smithi028.log: gzip -5 --verbose -- /var/log/ceph/5622b2fe-7402-11ef-bceb-c7b262605968/ceph-mgr.smithi028.azvyzy.log 2024-09-16T08:32:58.710 INFO:teuthology.orchestra.run.smithi028.stderr:gzip -5 --verbose -- /var/log/ceph/5622b2fe-7402-11ef-bceb-c7b262605968/ceph.audit.log 2024-09-16T08:32:58.710 INFO:teuthology.orchestra.run.smithi028.stderr:/var/log/ceph/5622b2fe-7402-11ef-bceb-c7b262605968/ceph-volume.log: /var/log/ceph/5622b2fe-7402-11ef-bceb-c7b262605968/ceph-mgr.smithi028.azvyzy.log: gzip -5 --verbose -- /var/log/ceph/5622b2fe-7402-11ef-bceb-c7b262605968/ceph-mds.foofs.smithi028.srtgev.log 2024-09-16T08:32:58.710 INFO:teuthology.orchestra.run.smithi028.stderr:/var/log/ceph/5622b2fe-7402-11ef-bceb-c7b262605968/ceph.audit.log: gzip -5 --verbose -- /var/log/ceph/5622b2fe-7402-11ef-bceb-c7b262605968/ceph.cephadm.log 2024-09-16T08:32:58.711 INFO:teuthology.orchestra.run.smithi028.stderr:/var/log/ceph/5622b2fe-7402-11ef-bceb-c7b262605968/ceph-mds.foofs.smithi028.srtgev.log: gzip -5 --verbose -- /var/log/ceph/5622b2fe-7402-11ef-bceb-c7b262605968/ceph-osd.2.log 2024-09-16T08:32:58.711 INFO:teuthology.orchestra.run.smithi028.stderr:/var/log/ceph/5622b2fe-7402-11ef-bceb-c7b262605968/ceph.cephadm.log: 85.5% -- replaced with /var/log/ceph/5622b2fe-7402-11ef-bceb-c7b262605968/ceph.cephadm.log.gz 2024-09-16T08:32:58.711 INFO:teuthology.orchestra.run.smithi028.stderr:gzip -5 --verbose -- /var/log/ceph/5622b2fe-7402-11ef-bceb-c7b262605968/ceph.log 2024-09-16T08:32:58.711 INFO:teuthology.orchestra.run.smithi144.stderr: 91.5% -- replaced with /var/log/ceph/5622b2fe-7402-11ef-bceb-c7b262605968/ceph.audit.log.gz 2024-09-16T08:32:58.712 INFO:teuthology.orchestra.run.smithi028.stderr:/var/log/ceph/5622b2fe-7402-11ef-bceb-c7b262605968/ceph-osd.2.log: 91.4% -- replaced with /var/log/ceph/5622b2fe-7402-11ef-bceb-c7b262605968/ceph.audit.log.gz 2024-09-16T08:32:58.718 INFO:teuthology.orchestra.run.smithi144.stderr: 91.3% -- replaced with /var/log/ceph/5622b2fe-7402-11ef-bceb-c7b262605968/ceph-volume.log.gz 2024-09-16T08:32:58.720 INFO:teuthology.orchestra.run.smithi144.stderr:gzip -5 --verbose -- /var/log/ceph/5622b2fe-7402-11ef-bceb-c7b262605968/ceph.log 2024-09-16T08:32:58.720 INFO:teuthology.orchestra.run.smithi144.stderr:/var/log/ceph/5622b2fe-7402-11ef-bceb-c7b262605968/ceph-osd.5.log: gzip -5 --verbose -- /var/log/ceph/5622b2fe-7402-11ef-bceb-c7b262605968/ceph-mds.foofs.smithi144.oozgsi.log 2024-09-16T08:32:58.720 INFO:teuthology.orchestra.run.smithi144.stderr:/var/log/ceph/5622b2fe-7402-11ef-bceb-c7b262605968/ceph.log: gzip -5 --verbose -- /var/log/ceph/cephadm.log 2024-09-16T08:32:58.722 INFO:teuthology.orchestra.run.smithi144.stderr:/var/log/ceph/5622b2fe-7402-11ef-bceb-c7b262605968/ceph-mds.foofs.smithi144.oozgsi.log: 88.4% -- replaced with /var/log/ceph/5622b2fe-7402-11ef-bceb-c7b262605968/ceph-mds.foofs.smithi144.oozgsi.log.gz 2024-09-16T08:32:58.722 INFO:teuthology.orchestra.run.smithi028.stderr:gzip -5 --verbose -- /var/log/ceph/5622b2fe-7402-11ef-bceb-c7b262605968/ceph-osd.3.log 2024-09-16T08:32:58.722 INFO:teuthology.orchestra.run.smithi028.stderr:/var/log/ceph/5622b2fe-7402-11ef-bceb-c7b262605968/ceph.log: 91.1% -- replaced with /var/log/ceph/5622b2fe-7402-11ef-bceb-c7b262605968/ceph-volume.log.gz 2024-09-16T08:32:58.723 INFO:teuthology.orchestra.run.smithi144.stderr:/var/log/ceph/cephadm.log: 86.9% -- replaced with /var/log/ceph/5622b2fe-7402-11ef-bceb-c7b262605968/ceph.log.gz 2024-09-16T08:32:58.724 INFO:teuthology.orchestra.run.smithi028.stderr: 86.9% -- replaced with /var/log/ceph/5622b2fe-7402-11ef-bceb-c7b262605968/ceph.log.gz 2024-09-16T08:32:58.729 INFO:teuthology.orchestra.run.smithi144.stderr: 89.9% -- replaced with /var/log/ceph/cephadm.log.gz 2024-09-16T08:32:58.732 INFO:teuthology.orchestra.run.smithi028.stderr: 91.3% -- replaced with /var/log/ceph/5622b2fe-7402-11ef-bceb-c7b262605968/ceph-mds.foofs.smithi028.srtgev.log.gz 2024-09-16T08:32:58.733 INFO:teuthology.orchestra.run.smithi028.stderr:gzip -5 --verbose -- /var/log/ceph/cephadm.log 2024-09-16T08:32:58.774 INFO:teuthology.orchestra.run.smithi028.stderr:/var/log/ceph/5622b2fe-7402-11ef-bceb-c7b262605968/ceph-osd.3.log: /var/log/ceph/cephadm.log: 91.7% -- replaced with /var/log/ceph/cephadm.log.gz 2024-09-16T08:32:58.776 INFO:teuthology.orchestra.run.smithi144.stderr: 89.8% -- replaced with /var/log/ceph/5622b2fe-7402-11ef-bceb-c7b262605968/ceph-mgr.smithi144.utzszj.log.gz 2024-09-16T08:32:58.981 INFO:teuthology.orchestra.run.smithi028.stderr: 88.7% -- replaced with /var/log/ceph/5622b2fe-7402-11ef-bceb-c7b262605968/ceph-mgr.smithi028.azvyzy.log.gz 2024-09-16T08:32:59.015 INFO:teuthology.orchestra.run.smithi144.stderr: 92.5% -- replaced with /var/log/ceph/5622b2fe-7402-11ef-bceb-c7b262605968/ceph-mon.smithi144.log.gz 2024-09-16T08:33:00.705 INFO:teuthology.orchestra.run.smithi028.stderr: 91.9% -- replaced with /var/log/ceph/5622b2fe-7402-11ef-bceb-c7b262605968/ceph-mon.smithi028.log.gz 2024-09-16T08:33:11.921 INFO:teuthology.orchestra.run.smithi028.stderr: 94.5% -- replaced with /var/log/ceph/5622b2fe-7402-11ef-bceb-c7b262605968/ceph-osd.2.log.gz 2024-09-16T08:33:12.264 INFO:teuthology.orchestra.run.smithi144.stderr: 93.9% -- replaced with /var/log/ceph/5622b2fe-7402-11ef-bceb-c7b262605968/ceph-osd.4.log.gz 2024-09-16T08:33:14.514 INFO:teuthology.orchestra.run.smithi144.stderr: 94.2% -- replaced with /var/log/ceph/5622b2fe-7402-11ef-bceb-c7b262605968/ceph-osd.6.log.gz 2024-09-16T08:33:15.402 INFO:teuthology.orchestra.run.smithi028.stderr: 94.1% -- replaced with /var/log/ceph/5622b2fe-7402-11ef-bceb-c7b262605968/ceph-osd.0.log.gz 2024-09-16T08:33:17.930 INFO:teuthology.orchestra.run.smithi028.stderr: 94.2% -- replaced with /var/log/ceph/5622b2fe-7402-11ef-bceb-c7b262605968/ceph-osd.1.log.gz 2024-09-16T08:33:17.960 INFO:teuthology.orchestra.run.smithi028.stderr: 94.1% -- replaced with /var/log/ceph/5622b2fe-7402-11ef-bceb-c7b262605968/ceph-osd.3.log.gz 2024-09-16T08:33:17.962 INFO:teuthology.orchestra.run.smithi028.stderr: 2024-09-16T08:33:17.962 INFO:teuthology.orchestra.run.smithi028.stderr:real 0m19.269s 2024-09-16T08:33:17.963 INFO:teuthology.orchestra.run.smithi028.stderr:user 1m8.127s 2024-09-16T08:33:17.963 INFO:teuthology.orchestra.run.smithi028.stderr:sys 0m2.491s 2024-09-16T08:33:18.914 INFO:teuthology.orchestra.run.smithi144.stderr: 94.2% -- replaced with /var/log/ceph/5622b2fe-7402-11ef-bceb-c7b262605968/ceph-osd.5.log.gz 2024-09-16T08:33:21.871 INFO:teuthology.orchestra.run.smithi144.stderr: 94.1% -- replaced with /var/log/ceph/5622b2fe-7402-11ef-bceb-c7b262605968/ceph-osd.7.log.gz 2024-09-16T08:33:21.873 INFO:teuthology.orchestra.run.smithi144.stderr: 2024-09-16T08:33:21.873 INFO:teuthology.orchestra.run.smithi144.stderr:real 0m23.179s 2024-09-16T08:33:21.873 INFO:teuthology.orchestra.run.smithi144.stderr:user 1m10.191s 2024-09-16T08:33:21.873 INFO:teuthology.orchestra.run.smithi144.stderr:sys 0m2.797s 2024-09-16T08:33:21.874 INFO:tasks.cephadm:Archiving logs... 2024-09-16T08:33:21.874 DEBUG:teuthology.misc:Transferring archived files from smithi028:/var/log/ceph to /home/teuthworker/archive/gabrioux-2024-09-16_07:44:22-orch:cephadm-wip-guits-main-2024-09-13-1248-distro-default-smithi/7906916/remote/smithi028/log 2024-09-16T08:33:21.875 DEBUG:teuthology.orchestra.run.smithi028:> sudo tar c -f - -C /var/log/ceph -- . 2024-09-16T08:33:31.388 DEBUG:teuthology.misc:Transferring archived files from smithi144:/var/log/ceph to /home/teuthworker/archive/gabrioux-2024-09-16_07:44:22-orch:cephadm-wip-guits-main-2024-09-13-1248-distro-default-smithi/7906916/remote/smithi144/log 2024-09-16T08:33:31.389 DEBUG:teuthology.orchestra.run.smithi144:> sudo tar c -f - -C /var/log/ceph -- . 2024-09-16T08:33:42.130 INFO:tasks.cephadm:Removing cluster... 2024-09-16T08:33:42.131 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 5622b2fe-7402-11ef-bceb-c7b262605968 --force 2024-09-16T08:33:42.506 DEBUG:teuthology.orchestra.run.smithi144:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 5622b2fe-7402-11ef-bceb-c7b262605968 --force 2024-09-16T08:33:42.881 INFO:tasks.cephadm:Removing cephadm ... 2024-09-16T08:33:42.881 DEBUG:teuthology.orchestra.run.smithi028:> rm -rf /home/ubuntu/cephtest/cephadm 2024-09-16T08:33:42.898 DEBUG:teuthology.orchestra.run.smithi144:> rm -rf /home/ubuntu/cephtest/cephadm 2024-09-16T08:33:42.913 INFO:tasks.cephadm:Teardown complete 2024-09-16T08:33:42.914 DEBUG:teuthology.run_tasks:Unwinding manager clock 2024-09-16T08:33:42.925 INFO:teuthology.task.clock:Checking final clock skew... 2024-09-16T08:33:42.925 DEBUG:teuthology.orchestra.run.smithi028:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-09-16T08:33:42.940 DEBUG:teuthology.orchestra.run.smithi144:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-09-16T08:33:42.953 INFO:teuthology.orchestra.run.smithi028.stderr:bash: line 1: ntpq: command not found 2024-09-16T08:33:42.957 INFO:teuthology.orchestra.run.smithi028.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-09-16T08:33:42.958 INFO:teuthology.orchestra.run.smithi028.stdout:=============================================================================== 2024-09-16T08:33:42.958 INFO:teuthology.orchestra.run.smithi028.stdout:^* hv01.front.sepia.ceph.com 3 6 377 45 +1222us[+1264us] +/- 54ms 2024-09-16T08:33:42.958 INFO:teuthology.orchestra.run.smithi028.stdout:^+ hv02.front.sepia.ceph.com 3 6 377 51 -2347us[-2305us] +/- 55ms 2024-09-16T08:33:42.958 INFO:teuthology.orchestra.run.smithi028.stdout:^+ hv03.front.sepia.ceph.com 3 6 377 49 +1341us[+1383us] +/- 64ms 2024-09-16T08:33:42.958 INFO:teuthology.orchestra.run.smithi028.stdout:^? hv04.front.sepia.ceph.com 0 9 0 - +0ns[ +0ns] +/- 0ns 2024-09-16T08:33:42.968 INFO:teuthology.orchestra.run.smithi144.stderr:bash: line 1: ntpq: command not found 2024-09-16T08:33:42.973 INFO:teuthology.orchestra.run.smithi144.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-09-16T08:33:42.973 INFO:teuthology.orchestra.run.smithi144.stdout:=============================================================================== 2024-09-16T08:33:42.973 INFO:teuthology.orchestra.run.smithi144.stdout:^* hv01.front.sepia.ceph.com 3 6 377 48 +1212us[+1242us] +/- 54ms 2024-09-16T08:33:42.973 INFO:teuthology.orchestra.run.smithi144.stdout:^+ hv02.front.sepia.ceph.com 3 6 377 51 -2337us[-2308us] +/- 56ms 2024-09-16T08:33:42.973 INFO:teuthology.orchestra.run.smithi144.stdout:^+ hv03.front.sepia.ceph.com 3 6 377 51 +1340us[+1370us] +/- 64ms 2024-09-16T08:33:42.973 INFO:teuthology.orchestra.run.smithi144.stdout:^? hv04.front.sepia.ceph.com 0 9 0 - +0ns[ +0ns] +/- 0ns 2024-09-16T08:33:42.973 DEBUG:teuthology.run_tasks:Unwinding manager ansible.cephlab 2024-09-16T08:33:42.982 INFO:teuthology.task.ansible:Skipping ansible cleanup... 2024-09-16T08:33:42.983 DEBUG:teuthology.run_tasks:Unwinding manager selinux 2024-09-16T08:33:43.042 DEBUG:teuthology.orchestra.run.smithi028:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2024-09-16T08:33:43.123 DEBUG:teuthology.orchestra.run.smithi144:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2024-09-16T08:33:43.193 DEBUG:teuthology.orchestra.run.smithi028:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e context=system_u:system_r:getty_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 2024-09-16T08:33:43.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-16T08:33:43.223 DEBUG:teuthology.orchestra.run.smithi144:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e context=system_u:system_r:getty_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 2024-09-16T08:33:43.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-16T08:33:43.251 DEBUG:teuthology.run_tasks:Unwinding manager pcp 2024-09-16T08:33:43.261 DEBUG:teuthology.run_tasks:Unwinding manager internal.timer 2024-09-16T08:33:43.271 INFO:teuthology.task.internal:Duration was 1963.045252 seconds 2024-09-16T08:33:43.271 DEBUG:teuthology.run_tasks:Unwinding manager internal.syslog 2024-09-16T08:33:43.280 INFO:teuthology.task.internal.syslog:Shutting down syslog monitoring... 2024-09-16T08:33:43.281 DEBUG:teuthology.orchestra.run.smithi028:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2024-09-16T08:33:43.283 DEBUG:teuthology.orchestra.run.smithi144:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2024-09-16T08:33:43.321 INFO:teuthology.orchestra.run.smithi028.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-09-16T08:33:43.331 INFO:teuthology.orchestra.run.smithi144.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-09-16T08:33:43.629 INFO:teuthology.task.internal.syslog:Checking logs for errors... 2024-09-16T08:33:43.629 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi028.front.sepia.ceph.com 2024-09-16T08:33:43.629 DEBUG:teuthology.orchestra.run.smithi028:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2024-09-16T08:33:43.651 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi144.front.sepia.ceph.com 2024-09-16T08:33:43.651 DEBUG:teuthology.orchestra.run.smithi144:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2024-09-16T08:33:43.697 INFO:teuthology.task.internal.syslog:Compressing syslogs... 2024-09-16T08:33:43.697 DEBUG:teuthology.orchestra.run.smithi028:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-09-16T08:33:43.700 DEBUG:teuthology.orchestra.run.smithi144:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-09-16T08:33:43.763 INFO:teuthology.task.internal.syslog:Gathering journactl -b0... 2024-09-16T08:33:43.763 DEBUG:teuthology.orchestra.run.smithi028:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2024-09-16T08:33:43.768 DEBUG:teuthology.orchestra.run.smithi144:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2024-09-16T08:33:58.133 DEBUG:teuthology.run_tasks:Unwinding manager internal.sudo 2024-09-16T08:33:58.143 INFO:teuthology.task.internal:Restoring /etc/sudoers... 2024-09-16T08:33:58.144 DEBUG:teuthology.orchestra.run.smithi028:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2024-09-16T08:33:58.170 DEBUG:teuthology.orchestra.run.smithi144:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2024-09-16T08:33:58.200 DEBUG:teuthology.run_tasks:Unwinding manager internal.coredump 2024-09-16T08:33:58.212 DEBUG:teuthology.orchestra.run.smithi028:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2024-09-16T08:33:58.214 DEBUG:teuthology.orchestra.run.smithi144:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2024-09-16T08:33:58.236 INFO:teuthology.orchestra.run.smithi028.stdout:kernel.core_pattern = core 2024-09-16T08:33:58.268 INFO:teuthology.orchestra.run.smithi144.stdout:kernel.core_pattern = core 2024-09-16T08:33:58.288 DEBUG:teuthology.orchestra.run.smithi028:> test -e /home/ubuntu/cephtest/archive/coredump 2024-09-16T08:33:58.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-16T08:33:58.313 DEBUG:teuthology.orchestra.run.smithi144:> test -e /home/ubuntu/cephtest/archive/coredump 2024-09-16T08:33:58.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-16T08:33:58.345 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive 2024-09-16T08:33:58.356 INFO:teuthology.task.internal:Transferring archived files... 2024-09-16T08:33:58.357 DEBUG:teuthology.misc:Transferring archived files from smithi028:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/gabrioux-2024-09-16_07:44:22-orch:cephadm-wip-guits-main-2024-09-13-1248-distro-default-smithi/7906916/remote/smithi028 2024-09-16T08:33:58.357 DEBUG:teuthology.orchestra.run.smithi028:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2024-09-16T08:33:58.421 DEBUG:teuthology.misc:Transferring archived files from smithi144:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/gabrioux-2024-09-16_07:44:22-orch:cephadm-wip-guits-main-2024-09-13-1248-distro-default-smithi/7906916/remote/smithi144 2024-09-16T08:33:58.421 DEBUG:teuthology.orchestra.run.smithi144:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2024-09-16T08:33:58.464 INFO:teuthology.task.internal:Removing archive directory... 2024-09-16T08:33:58.464 DEBUG:teuthology.orchestra.run.smithi028:> rm -rf -- /home/ubuntu/cephtest/archive 2024-09-16T08:33:58.467 DEBUG:teuthology.orchestra.run.smithi144:> rm -rf -- /home/ubuntu/cephtest/archive 2024-09-16T08:33:58.513 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive_upload 2024-09-16T08:33:58.525 INFO:teuthology.task.internal:Not uploading archives. 2024-09-16T08:33:58.525 DEBUG:teuthology.run_tasks:Unwinding manager internal.base 2024-09-16T08:33:58.535 INFO:teuthology.task.internal:Tidying up after the test... 2024-09-16T08:33:58.536 DEBUG:teuthology.orchestra.run.smithi028:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2024-09-16T08:33:58.538 DEBUG:teuthology.orchestra.run.smithi144:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2024-09-16T08:33:58.551 INFO:teuthology.orchestra.run.smithi028.stdout: 265324 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Sep 16 08:33 /home/ubuntu/cephtest 2024-09-16T08:33:58.570 INFO:teuthology.orchestra.run.smithi144.stdout: 265324 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Sep 16 08:33 /home/ubuntu/cephtest 2024-09-16T08:33:58.571 DEBUG:teuthology.run_tasks:Unwinding manager kernel 2024-09-16T08:33:58.582 DEBUG:teuthology.run_tasks:Unwinding manager console_log 2024-09-16T08:33:58.608 INFO:teuthology.run:Summary data: description: orch:cephadm/mgr-nfs-upgrade/{0-centos_9.stream 1-bootstrap/17.2.0 1-start 2-nfs 3-upgrade-with-workload 4-final} duration: 1963.045252084732 owner: scheduled_gabrioux@teuthology success: true 2024-09-16T08:33:58.608 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2024-09-16T08:33:58.687 INFO:teuthology.run:pass